diff options
Diffstat (limited to 'sphinx/ext/autodoc')
| -rw-r--r-- | sphinx/ext/autodoc/__init__.py | 2846 | ||||
| -rw-r--r-- | sphinx/ext/autodoc/directive.py | 160 | ||||
| -rw-r--r-- | sphinx/ext/autodoc/importer.py | 301 | ||||
| -rw-r--r-- | sphinx/ext/autodoc/mock.py | 193 | ||||
| -rw-r--r-- | sphinx/ext/autodoc/preserve_defaults.py | 127 | ||||
| -rw-r--r-- | sphinx/ext/autodoc/type_comment.py | 131 | ||||
| -rw-r--r-- | sphinx/ext/autodoc/typehints.py | 213 |
7 files changed, 3971 insertions, 0 deletions
diff --git a/sphinx/ext/autodoc/__init__.py b/sphinx/ext/autodoc/__init__.py new file mode 100644 index 0000000..93df64b --- /dev/null +++ b/sphinx/ext/autodoc/__init__.py @@ -0,0 +1,2846 @@ +"""Extension to create automatic documentation from code docstrings. + +Automatically insert docstrings for functions, classes or whole modules into +the doctree, thus avoiding duplication between docstrings and documentation +for those who like elaborate docstrings. +""" + +import re +import warnings +from inspect import Parameter, Signature +from types import ModuleType +from typing import (TYPE_CHECKING, Any, Callable, Dict, Iterator, List, Optional, Sequence, + Set, Tuple, Type, TypeVar, Union) + +from docutils.statemachine import StringList + +import sphinx +from sphinx.application import Sphinx +from sphinx.config import ENUM, Config +from sphinx.deprecation import RemovedInSphinx60Warning +from sphinx.environment import BuildEnvironment +from sphinx.ext.autodoc.importer import (get_class_members, get_object_members, import_module, + import_object) +from sphinx.ext.autodoc.mock import ismock, mock, undecorate +from sphinx.locale import _, __ +from sphinx.pycode import ModuleAnalyzer, PycodeError +from sphinx.util import inspect, logging +from sphinx.util.docstrings import prepare_docstring, separate_metadata +from sphinx.util.inspect import (evaluate_signature, getdoc, object_description, safe_getattr, + stringify_signature) +from sphinx.util.typing import OptionSpec, get_type_hints, restify +from sphinx.util.typing import stringify as stringify_typehint + +if TYPE_CHECKING: + from sphinx.ext.autodoc.directive import DocumenterBridge + +logger = logging.getLogger(__name__) + + +# This type isn't exposed directly in any modules, but can be found +# here in most Python versions +MethodDescriptorType = type(type.__subclasses__) + + +#: extended signature RE: with explicit module name separated by :: +py_ext_sig_re = re.compile( + r'''^ ([\w.]+::)? # explicit module name + ([\w.]+\.)? # module and/or class name(s) + (\w+) \s* # thing name + (?: \((.*)\) # optional: arguments + (?:\s* -> \s* (.*))? # return annotation + )? $ # and nothing more + ''', re.VERBOSE) +special_member_re = re.compile(r'^__\S+__$') + + +def identity(x: Any) -> Any: + return x + + +class _All: + """A special value for :*-members: that matches to any member.""" + + def __contains__(self, item: Any) -> bool: + return True + + def append(self, item: Any) -> None: + pass # nothing + + +class _Empty: + """A special value for :exclude-members: that never matches to any member.""" + + def __contains__(self, item: Any) -> bool: + return False + + +ALL = _All() +EMPTY = _Empty() +UNINITIALIZED_ATTR = object() +INSTANCEATTR = object() +SLOTSATTR = object() + + +def members_option(arg: Any) -> Union[object, List[str]]: + """Used to convert the :members: option to auto directives.""" + if arg in (None, True): + return ALL + elif arg is False: + return None + else: + return [x.strip() for x in arg.split(',') if x.strip()] + + +def exclude_members_option(arg: Any) -> Union[object, Set[str]]: + """Used to convert the :exclude-members: option.""" + if arg in (None, True): + return EMPTY + return {x.strip() for x in arg.split(',') if x.strip()} + + +def inherited_members_option(arg: Any) -> Set[str]: + """Used to convert the :inherited-members: option to auto directives.""" + if arg in (None, True): + return {'object'} + elif arg: + return {x.strip() for x in arg.split(',')} + else: + return set() + + +def member_order_option(arg: Any) -> Optional[str]: + """Used to convert the :member-order: option to auto directives.""" + if arg in (None, True): + return None + elif arg in ('alphabetical', 'bysource', 'groupwise'): + return arg + else: + raise ValueError(__('invalid value for member-order option: %s') % arg) + + +def class_doc_from_option(arg: Any) -> Optional[str]: + """Used to convert the :class-doc-from: option to autoclass directives.""" + if arg in ('both', 'class', 'init'): + return arg + else: + raise ValueError(__('invalid value for class-doc-from option: %s') % arg) + + +SUPPRESS = object() + + +def annotation_option(arg: Any) -> Any: + if arg in (None, True): + # suppress showing the representation of the object + return SUPPRESS + else: + return arg + + +def bool_option(arg: Any) -> bool: + """Used to convert flag options to auto directives. (Instead of + directives.flag(), which returns None). + """ + return True + + +def merge_members_option(options: Dict) -> None: + """Merge :private-members: and :special-members: options to the + :members: option. + """ + if options.get('members') is ALL: + # merging is not needed when members: ALL + return + + members = options.setdefault('members', []) + for key in {'private-members', 'special-members'}: + if key in options and options[key] not in (ALL, None): + for member in options[key]: + if member not in members: + members.append(member) + + +# Some useful event listener factories for autodoc-process-docstring. + +def cut_lines(pre: int, post: int = 0, what: str = None) -> Callable: + """Return a listener that removes the first *pre* and last *post* + lines of every docstring. If *what* is a sequence of strings, + only docstrings of a type in *what* will be processed. + + Use like this (e.g. in the ``setup()`` function of :file:`conf.py`):: + + from sphinx.ext.autodoc import cut_lines + app.connect('autodoc-process-docstring', cut_lines(4, what=['module'])) + + This can (and should) be used in place of :confval:`automodule_skip_lines`. + """ + def process(app: Sphinx, what_: str, name: str, obj: Any, options: Any, lines: List[str] + ) -> None: + if what and what_ not in what: + return + del lines[:pre] + if post: + # remove one trailing blank line. + if lines and not lines[-1]: + lines.pop(-1) + del lines[-post:] + # make sure there is a blank line at the end + if lines and lines[-1]: + lines.append('') + return process + + +def between(marker: str, what: Sequence[str] = None, keepempty: bool = False, + exclude: bool = False) -> Callable: + """Return a listener that either keeps, or if *exclude* is True excludes, + lines between lines that match the *marker* regular expression. If no line + matches, the resulting docstring would be empty, so no change will be made + unless *keepempty* is true. + + If *what* is a sequence of strings, only docstrings of a type in *what* will + be processed. + """ + marker_re = re.compile(marker) + + def process(app: Sphinx, what_: str, name: str, obj: Any, options: Any, lines: List[str] + ) -> None: + if what and what_ not in what: + return + deleted = 0 + delete = not exclude + orig_lines = lines[:] + for i, line in enumerate(orig_lines): + if delete: + lines.pop(i - deleted) + deleted += 1 + if marker_re.match(line): + delete = not delete + if delete: + lines.pop(i - deleted) + deleted += 1 + if not lines and not keepempty: + lines[:] = orig_lines + # make sure there is a blank line at the end + if lines and lines[-1]: + lines.append('') + return process + + +# This class is used only in ``sphinx.ext.autodoc.directive``, +# But we define this class here to keep compatibility (see #4538) +class Options(dict): + """A dict/attribute hybrid that returns None on nonexisting keys.""" + def copy(self) -> "Options": + return Options(super().copy()) + + def __getattr__(self, name: str) -> Any: + try: + return self[name.replace('_', '-')] + except KeyError: + return None + + +class ObjectMember(tuple): + """A member of object. + + This is used for the result of `Documenter.get_object_members()` to + represent each member of the object. + + .. Note:: + + An instance of this class behaves as a tuple of (name, object) + for compatibility to old Sphinx. The behavior will be dropped + in the future. Therefore extensions should not use the tuple + interface. + """ + + def __new__(cls, name: str, obj: Any, **kwargs: Any) -> Any: + return super().__new__(cls, (name, obj)) # type: ignore + + def __init__(self, name: str, obj: Any, docstring: Optional[str] = None, + class_: Any = None, skipped: bool = False) -> None: + self.__name__ = name + self.object = obj + self.docstring = docstring + self.skipped = skipped + self.class_ = class_ + + +ObjectMembers = Union[List[ObjectMember], List[Tuple[str, Any]]] + + +class Documenter: + """ + A Documenter knows how to autodocument a single object type. When + registered with the AutoDirective, it will be used to document objects + of that type when needed by autodoc. + + Its *objtype* attribute selects what auto directive it is assigned to + (the directive name is 'auto' + objtype), and what directive it generates + by default, though that can be overridden by an attribute called + *directivetype*. + + A Documenter has an *option_spec* that works like a docutils directive's; + in fact, it will be used to parse an auto directive's options that matches + the Documenter. + """ + #: name by which the directive is called (auto...) and the default + #: generated directive name + objtype = 'object' + #: indentation by which to indent the directive content + content_indent = ' ' + #: priority if multiple documenters return True from can_document_member + priority = 0 + #: order if autodoc_member_order is set to 'groupwise' + member_order = 0 + #: true if the generated content may contain titles + titles_allowed = False + + option_spec: OptionSpec = { + 'noindex': bool_option + } + + def get_attr(self, obj: Any, name: str, *defargs: Any) -> Any: + """getattr() override for types such as Zope interfaces.""" + return autodoc_attrgetter(self.env.app, obj, name, *defargs) + + @classmethod + def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any + ) -> bool: + """Called to see if a member can be documented by this Documenter.""" + raise NotImplementedError('must be implemented in subclasses') + + def __init__(self, directive: "DocumenterBridge", name: str, indent: str = '') -> None: + self.directive = directive + self.config: Config = directive.env.config + self.env: BuildEnvironment = directive.env + self.options = directive.genopt + self.name = name + self.indent = indent + # the module and object path within the module, and the fully + # qualified name (all set after resolve_name succeeds) + self.modname: str = None + self.module: ModuleType = None + self.objpath: List[str] = None + self.fullname: str = None + # extra signature items (arguments and return annotation, + # also set after resolve_name succeeds) + self.args: str = None + self.retann: str = None + # the object to document (set after import_object succeeds) + self.object: Any = None + self.object_name: str = None + # the parent/owner of the object to document + self.parent: Any = None + # the module analyzer to get at attribute docs, or None + self.analyzer: ModuleAnalyzer = None + + @property + def documenters(self) -> Dict[str, Type["Documenter"]]: + """Returns registered Documenter classes""" + return self.env.app.registry.documenters + + def add_line(self, line: str, source: str, *lineno: int) -> None: + """Append one line of generated reST to the output.""" + if line.strip(): # not a blank line + self.directive.result.append(self.indent + line, source, *lineno) + else: + self.directive.result.append('', source, *lineno) + + def resolve_name(self, modname: str, parents: Any, path: str, base: Any + ) -> Tuple[str, List[str]]: + """Resolve the module and name of the object to document given by the + arguments and the current module/class. + + Must return a pair of the module name and a chain of attributes; for + example, it would return ``('zipfile', ['ZipFile', 'open'])`` for the + ``zipfile.ZipFile.open`` method. + """ + raise NotImplementedError('must be implemented in subclasses') + + def parse_name(self) -> bool: + """Determine what module to import and what attribute to document. + + Returns True and sets *self.modname*, *self.objpath*, *self.fullname*, + *self.args* and *self.retann* if parsing and resolving was successful. + """ + # first, parse the definition -- auto directives for classes and + # functions can contain a signature which is then used instead of + # an autogenerated one + try: + matched = py_ext_sig_re.match(self.name) + explicit_modname, path, base, args, retann = matched.groups() + except AttributeError: + logger.warning(__('invalid signature for auto%s (%r)') % (self.objtype, self.name), + type='autodoc') + return False + + # support explicit module and class name separation via :: + if explicit_modname is not None: + modname = explicit_modname[:-2] + parents = path.rstrip('.').split('.') if path else [] + else: + modname = None + parents = [] + + with mock(self.config.autodoc_mock_imports): + self.modname, self.objpath = self.resolve_name(modname, parents, path, base) + + if not self.modname: + return False + + self.args = args + self.retann = retann + self.fullname = ((self.modname or '') + + ('.' + '.'.join(self.objpath) if self.objpath else '')) + return True + + def import_object(self, raiseerror: bool = False) -> bool: + """Import the object given by *self.modname* and *self.objpath* and set + it as *self.object*. + + Returns True if successful, False if an error occurred. + """ + with mock(self.config.autodoc_mock_imports): + try: + ret = import_object(self.modname, self.objpath, self.objtype, + attrgetter=self.get_attr, + warningiserror=self.config.autodoc_warningiserror) + self.module, self.parent, self.object_name, self.object = ret + if ismock(self.object): + self.object = undecorate(self.object) + return True + except ImportError as exc: + if raiseerror: + raise + else: + logger.warning(exc.args[0], type='autodoc', subtype='import_object') + self.env.note_reread() + return False + + def get_real_modname(self) -> str: + """Get the real module name of an object to document. + + It can differ from the name of the module through which the object was + imported. + """ + return self.get_attr(self.object, '__module__', None) or self.modname + + def check_module(self) -> bool: + """Check if *self.object* is really defined in the module given by + *self.modname*. + """ + if self.options.imported_members: + return True + + subject = inspect.unpartial(self.object) + modname = self.get_attr(subject, '__module__', None) + if modname and modname != self.modname: + return False + return True + + def format_args(self, **kwargs: Any) -> str: + """Format the argument signature of *self.object*. + + Should return None if the object does not have a signature. + """ + return None + + def format_name(self) -> str: + """Format the name of *self.object*. + + This normally should be something that can be parsed by the generated + directive, but doesn't need to be (Sphinx will display it unparsed + then). + """ + # normally the name doesn't contain the module (except for module + # directives of course) + return '.'.join(self.objpath) or self.modname + + def _call_format_args(self, **kwargs: Any) -> str: + if kwargs: + try: + return self.format_args(**kwargs) + except TypeError: + # avoid chaining exceptions, by putting nothing here + pass + + # retry without arguments for old documenters + return self.format_args() + + def format_signature(self, **kwargs: Any) -> str: + """Format the signature (arguments and return annotation) of the object. + + Let the user process it via the ``autodoc-process-signature`` event. + """ + if self.args is not None: + # signature given explicitly + args = "(%s)" % self.args + retann = self.retann + else: + # try to introspect the signature + try: + retann = None + args = self._call_format_args(**kwargs) + if args: + matched = re.match(r'^(\(.*\))\s+->\s+(.*)$', args) + if matched: + args = matched.group(1) + retann = matched.group(2) + except Exception as exc: + logger.warning(__('error while formatting arguments for %s: %s'), + self.fullname, exc, type='autodoc') + args = None + + result = self.env.events.emit_firstresult('autodoc-process-signature', + self.objtype, self.fullname, + self.object, self.options, args, retann) + if result: + args, retann = result + + if args is not None: + return args + ((' -> %s' % retann) if retann else '') + else: + return '' + + def add_directive_header(self, sig: str) -> None: + """Add the directive header and options to the generated content.""" + domain = getattr(self, 'domain', 'py') + directive = getattr(self, 'directivetype', self.objtype) + name = self.format_name() + sourcename = self.get_sourcename() + + # one signature per line, indented by column + prefix = '.. %s:%s:: ' % (domain, directive) + for i, sig_line in enumerate(sig.split("\n")): + self.add_line('%s%s%s' % (prefix, name, sig_line), + sourcename) + if i == 0: + prefix = " " * len(prefix) + + if self.options.noindex: + self.add_line(' :noindex:', sourcename) + if self.objpath: + # Be explicit about the module, this is necessary since .. class:: + # etc. don't support a prepended module name + self.add_line(' :module: %s' % self.modname, sourcename) + + def get_doc(self) -> Optional[List[List[str]]]: + """Decode and return lines of the docstring(s) for the object. + + When it returns None, autodoc-process-docstring will not be called for this + object. + """ + docstring = getdoc(self.object, self.get_attr, self.config.autodoc_inherit_docstrings, + self.parent, self.object_name) + if docstring: + tab_width = self.directive.state.document.settings.tab_width + return [prepare_docstring(docstring, tab_width)] + return [] + + def process_doc(self, docstrings: List[List[str]]) -> Iterator[str]: + """Let the user process the docstrings before adding them.""" + for docstringlines in docstrings: + if self.env.app: + # let extensions preprocess docstrings + self.env.app.emit('autodoc-process-docstring', + self.objtype, self.fullname, self.object, + self.options, docstringlines) + + if docstringlines and docstringlines[-1] != '': + # append a blank line to the end of the docstring + docstringlines.append('') + + yield from docstringlines + + def get_sourcename(self) -> str: + if (inspect.safe_getattr(self.object, '__module__', None) and + inspect.safe_getattr(self.object, '__qualname__', None)): + # Get the correct location of docstring from self.object + # to support inherited methods + fullname = '%s.%s' % (self.object.__module__, self.object.__qualname__) + else: + fullname = self.fullname + + if self.analyzer: + return '%s:docstring of %s' % (self.analyzer.srcname, fullname) + else: + return 'docstring of %s' % fullname + + def add_content(self, more_content: Optional[StringList]) -> None: + """Add content from docstrings, attribute documentation and user.""" + docstring = True + + # set sourcename and add content from attribute documentation + sourcename = self.get_sourcename() + if self.analyzer: + attr_docs = self.analyzer.find_attr_docs() + if self.objpath: + key = ('.'.join(self.objpath[:-1]), self.objpath[-1]) + if key in attr_docs: + docstring = False + # make a copy of docstring for attributes to avoid cache + # the change of autodoc-process-docstring event. + docstrings = [list(attr_docs[key])] + + for i, line in enumerate(self.process_doc(docstrings)): + self.add_line(line, sourcename, i) + + # add content from docstrings + if docstring: + docstrings = self.get_doc() + if docstrings is None: + # Do not call autodoc-process-docstring on get_doc() returns None. + pass + else: + if not docstrings: + # append at least a dummy docstring, so that the event + # autodoc-process-docstring is fired and can add some + # content if desired + docstrings.append([]) + for i, line in enumerate(self.process_doc(docstrings)): + self.add_line(line, sourcename, i) + + # add additional content (e.g. from document), if present + if more_content: + for line, src in zip(more_content.data, more_content.items): + self.add_line(line, src[0], src[1]) + + def get_object_members(self, want_all: bool) -> Tuple[bool, ObjectMembers]: + """Return `(members_check_module, members)` where `members` is a + list of `(membername, member)` pairs of the members of *self.object*. + + If *want_all* is True, return all members. Else, only return those + members given by *self.options.members* (which may also be None). + """ + warnings.warn('The implementation of Documenter.get_object_members() will be ' + 'removed from Sphinx-6.0.', RemovedInSphinx60Warning) + members = get_object_members(self.object, self.objpath, self.get_attr, self.analyzer) + if not want_all: + if not self.options.members: + return False, [] # type: ignore + # specific members given + selected = [] + for name in self.options.members: + if name in members: + selected.append((name, members[name].value)) + else: + logger.warning(__('missing attribute %s in object %s') % + (name, self.fullname), type='autodoc') + return False, selected + elif self.options.inherited_members: + return False, [(m.name, m.value) for m in members.values()] + else: + return False, [(m.name, m.value) for m in members.values() + if m.directly_defined] + + def filter_members(self, members: ObjectMembers, want_all: bool + ) -> List[Tuple[str, Any, bool]]: + """Filter the given member list. + + Members are skipped if + + - they are private (except if given explicitly or the private-members + option is set) + - they are special methods (except if given explicitly or the + special-members option is set) + - they are undocumented (except if the undoc-members option is set) + + The user can override the skipping decision by connecting to the + ``autodoc-skip-member`` event. + """ + def is_filtered_inherited_member(name: str, obj: Any) -> bool: + inherited_members = self.options.inherited_members or set() + + if inspect.isclass(self.object): + for cls in self.object.__mro__: + if cls.__name__ in inherited_members and cls != self.object: + # given member is a member of specified *super class* + return True + elif name in cls.__dict__: + return False + elif name in self.get_attr(cls, '__annotations__', {}): + return False + elif isinstance(obj, ObjectMember) and obj.class_ is cls: + return False + + return False + + ret = [] + + # search for members in source code too + namespace = '.'.join(self.objpath) # will be empty for modules + + if self.analyzer: + attr_docs = self.analyzer.find_attr_docs() + else: + attr_docs = {} + + # process members and determine which to skip + for obj in members: + try: + membername, member = obj + # if isattr is True, the member is documented as an attribute + if member is INSTANCEATTR: + isattr = True + elif (namespace, membername) in attr_docs: + isattr = True + else: + isattr = False + + doc = getdoc(member, self.get_attr, self.config.autodoc_inherit_docstrings, + self.object, membername) + if not isinstance(doc, str): + # Ignore non-string __doc__ + doc = None + + # if the member __doc__ is the same as self's __doc__, it's just + # inherited and therefore not the member's doc + cls = self.get_attr(member, '__class__', None) + if cls: + cls_doc = self.get_attr(cls, '__doc__', None) + if cls_doc == doc: + doc = None + + if isinstance(obj, ObjectMember) and obj.docstring: + # hack for ClassDocumenter to inject docstring via ObjectMember + doc = obj.docstring + + doc, metadata = separate_metadata(doc) + has_doc = bool(doc) + + if 'private' in metadata: + # consider a member private if docstring has "private" metadata + isprivate = True + elif 'public' in metadata: + # consider a member public if docstring has "public" metadata + isprivate = False + else: + isprivate = membername.startswith('_') + + keep = False + if ismock(member) and (namespace, membername) not in attr_docs: + # mocked module or object + pass + elif (self.options.exclude_members and + membername in self.options.exclude_members): + # remove members given by exclude-members + keep = False + elif want_all and special_member_re.match(membername): + # special __methods__ + if (self.options.special_members and + membername in self.options.special_members): + if membername == '__doc__': + keep = False + elif is_filtered_inherited_member(membername, obj): + keep = False + else: + keep = has_doc or self.options.undoc_members + else: + keep = False + elif (namespace, membername) in attr_docs: + if want_all and isprivate: + if self.options.private_members is None: + keep = False + else: + keep = membername in self.options.private_members + else: + # keep documented attributes + keep = True + elif want_all and isprivate: + if has_doc or self.options.undoc_members: + if self.options.private_members is None: + keep = False + elif is_filtered_inherited_member(membername, obj): + keep = False + else: + keep = membername in self.options.private_members + else: + keep = False + else: + if (self.options.members is ALL and + is_filtered_inherited_member(membername, obj)): + keep = False + else: + # ignore undocumented members if :undoc-members: is not given + keep = has_doc or self.options.undoc_members + + if isinstance(obj, ObjectMember) and obj.skipped: + # forcedly skipped member (ex. a module attribute not defined in __all__) + keep = False + + # give the user a chance to decide whether this member + # should be skipped + if self.env.app: + # let extensions preprocess docstrings + skip_user = self.env.app.emit_firstresult( + 'autodoc-skip-member', self.objtype, membername, member, + not keep, self.options) + if skip_user is not None: + keep = not skip_user + except Exception as exc: + logger.warning(__('autodoc: failed to determine %s.%s (%r) to be documented, ' + 'the following exception was raised:\n%s'), + self.name, membername, member, exc, type='autodoc') + keep = False + + if keep: + ret.append((membername, member, isattr)) + + return ret + + def document_members(self, all_members: bool = False) -> None: + """Generate reST for member documentation. + + If *all_members* is True, document all members, else those given by + *self.options.members*. + """ + # set current namespace for finding members + self.env.temp_data['autodoc:module'] = self.modname + if self.objpath: + self.env.temp_data['autodoc:class'] = self.objpath[0] + + want_all = (all_members or + self.options.inherited_members or + self.options.members is ALL) + # find out which members are documentable + members_check_module, members = self.get_object_members(want_all) + + # document non-skipped members + memberdocumenters: List[Tuple[Documenter, bool]] = [] + for (mname, member, isattr) in self.filter_members(members, want_all): + classes = [cls for cls in self.documenters.values() + if cls.can_document_member(member, mname, isattr, self)] + if not classes: + # don't know how to document this member + continue + # prefer the documenter with the highest priority + classes.sort(key=lambda cls: cls.priority) + # give explicitly separated module name, so that members + # of inner classes can be documented + full_mname = self.modname + '::' + '.'.join(self.objpath + [mname]) + documenter = classes[-1](self.directive, full_mname, self.indent) + memberdocumenters.append((documenter, isattr)) + + member_order = self.options.member_order or self.config.autodoc_member_order + memberdocumenters = self.sort_members(memberdocumenters, member_order) + + for documenter, isattr in memberdocumenters: + documenter.generate( + all_members=True, real_modname=self.real_modname, + check_module=members_check_module and not isattr) + + # reset current objects + self.env.temp_data['autodoc:module'] = None + self.env.temp_data['autodoc:class'] = None + + def sort_members(self, documenters: List[Tuple["Documenter", bool]], + order: str) -> List[Tuple["Documenter", bool]]: + """Sort the given member list.""" + if order == 'groupwise': + # sort by group; alphabetically within groups + documenters.sort(key=lambda e: (e[0].member_order, e[0].name)) + elif order == 'bysource': + if self.analyzer: + # sort by source order, by virtue of the module analyzer + tagorder = self.analyzer.tagorder + + def keyfunc(entry: Tuple[Documenter, bool]) -> int: + fullname = entry[0].name.split('::')[1] + return tagorder.get(fullname, len(tagorder)) + documenters.sort(key=keyfunc) + else: + # Assume that member discovery order matches source order. + # This is a reasonable assumption in Python 3.6 and up, where + # module.__dict__ is insertion-ordered. + pass + else: # alphabetical + documenters.sort(key=lambda e: e[0].name) + + return documenters + + def generate(self, more_content: Optional[StringList] = None, real_modname: str = None, + check_module: bool = False, all_members: bool = False) -> None: + """Generate reST for the object given by *self.name*, and possibly for + its members. + + If *more_content* is given, include that content. If *real_modname* is + given, use that module name to find attribute docs. If *check_module* is + True, only generate if the object is defined in the module name it is + imported from. If *all_members* is True, document all members. + """ + if not self.parse_name(): + # need a module to import + logger.warning( + __('don\'t know which module to import for autodocumenting ' + '%r (try placing a "module" or "currentmodule" directive ' + 'in the document, or giving an explicit module name)') % + self.name, type='autodoc') + return + + # now, import the module and get object to document + if not self.import_object(): + return + + # If there is no real module defined, figure out which to use. + # The real module is used in the module analyzer to look up the module + # where the attribute documentation would actually be found in. + # This is used for situations where you have a module that collects the + # functions and classes of internal submodules. + guess_modname = self.get_real_modname() + self.real_modname: str = real_modname or guess_modname + + # try to also get a source code analyzer for attribute docs + try: + self.analyzer = ModuleAnalyzer.for_module(self.real_modname) + # parse right now, to get PycodeErrors on parsing (results will + # be cached anyway) + self.analyzer.find_attr_docs() + except PycodeError as exc: + logger.debug('[autodoc] module analyzer failed: %s', exc) + # no source file -- e.g. for builtin and C modules + self.analyzer = None + # at least add the module.__file__ as a dependency + if hasattr(self.module, '__file__') and self.module.__file__: + self.directive.record_dependencies.add(self.module.__file__) + else: + self.directive.record_dependencies.add(self.analyzer.srcname) + + if self.real_modname != guess_modname: + # Add module to dependency list if target object is defined in other module. + try: + analyzer = ModuleAnalyzer.for_module(guess_modname) + self.directive.record_dependencies.add(analyzer.srcname) + except PycodeError: + pass + + docstrings: List[str] = sum(self.get_doc() or [], []) + if ismock(self.object) and not docstrings: + logger.warning(__('A mocked object is detected: %r'), + self.name, type='autodoc') + + # check __module__ of object (for members not given explicitly) + if check_module: + if not self.check_module(): + return + + sourcename = self.get_sourcename() + + # make sure that the result starts with an empty line. This is + # necessary for some situations where another directive preprocesses + # reST and no starting newline is present + self.add_line('', sourcename) + + # format the object's signature, if any + try: + sig = self.format_signature() + except Exception as exc: + logger.warning(__('error while formatting signature for %s: %s'), + self.fullname, exc, type='autodoc') + return + + # generate the directive header and options, if applicable + self.add_directive_header(sig) + self.add_line('', sourcename) + + # e.g. the module directive doesn't have content + self.indent += self.content_indent + + # add all content (from docstrings, attribute docs etc.) + self.add_content(more_content) + + # document members, if possible + self.document_members(all_members) + + +class ModuleDocumenter(Documenter): + """ + Specialized Documenter subclass for modules. + """ + objtype = 'module' + content_indent = '' + titles_allowed = True + _extra_indent = ' ' + + option_spec: OptionSpec = { + 'members': members_option, 'undoc-members': bool_option, + 'noindex': bool_option, 'inherited-members': inherited_members_option, + 'show-inheritance': bool_option, 'synopsis': identity, + 'platform': identity, 'deprecated': bool_option, + 'member-order': member_order_option, 'exclude-members': exclude_members_option, + 'private-members': members_option, 'special-members': members_option, + 'imported-members': bool_option, 'ignore-module-all': bool_option, + 'no-value': bool_option, + } + + def __init__(self, *args: Any) -> None: + super().__init__(*args) + merge_members_option(self.options) + self.__all__: Optional[Sequence[str]] = None + + def add_content(self, more_content: Optional[StringList]) -> None: + old_indent = self.indent + self.indent += self._extra_indent + super().add_content(None) + self.indent = old_indent + if more_content: + for line, src in zip(more_content.data, more_content.items): + self.add_line(line, src[0], src[1]) + + @classmethod + def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any + ) -> bool: + # don't document submodules automatically + return False + + def resolve_name(self, modname: str, parents: Any, path: str, base: Any + ) -> Tuple[str, List[str]]: + if modname is not None: + logger.warning(__('"::" in automodule name doesn\'t make sense'), + type='autodoc') + return (path or '') + base, [] + + def parse_name(self) -> bool: + ret = super().parse_name() + if self.args or self.retann: + logger.warning(__('signature arguments or return annotation ' + 'given for automodule %s') % self.fullname, + type='autodoc') + return ret + + def import_object(self, raiseerror: bool = False) -> bool: + ret = super().import_object(raiseerror) + + try: + if not self.options.ignore_module_all: + self.__all__ = inspect.getall(self.object) + except ValueError as exc: + # invalid __all__ found. + logger.warning(__('__all__ should be a list of strings, not %r ' + '(in module %s) -- ignoring __all__') % + (exc.args[0], self.fullname), type='autodoc') + + return ret + + def add_directive_header(self, sig: str) -> None: + Documenter.add_directive_header(self, sig) + + sourcename = self.get_sourcename() + + # add some module-specific options + if self.options.synopsis: + self.add_line(' :synopsis: ' + self.options.synopsis, sourcename) + if self.options.platform: + self.add_line(' :platform: ' + self.options.platform, sourcename) + if self.options.deprecated: + self.add_line(' :deprecated:', sourcename) + + def get_module_members(self) -> Dict[str, ObjectMember]: + """Get members of target module.""" + if self.analyzer: + attr_docs = self.analyzer.attr_docs + else: + attr_docs = {} + + members: Dict[str, ObjectMember] = {} + for name in dir(self.object): + try: + value = safe_getattr(self.object, name, None) + if ismock(value): + value = undecorate(value) + docstring = attr_docs.get(('', name), []) + members[name] = ObjectMember(name, value, docstring="\n".join(docstring)) + except AttributeError: + continue + + # annotation only member (ex. attr: int) + for name in inspect.getannotations(self.object): + if name not in members: + docstring = attr_docs.get(('', name), []) + members[name] = ObjectMember(name, INSTANCEATTR, + docstring="\n".join(docstring)) + + return members + + def get_object_members(self, want_all: bool) -> Tuple[bool, ObjectMembers]: + members = self.get_module_members() + if want_all: + if self.__all__ is None: + # for implicit module members, check __module__ to avoid + # documenting imported objects + return True, list(members.values()) + else: + for member in members.values(): + if member.__name__ not in self.__all__: + member.skipped = True + + return False, list(members.values()) + else: + memberlist = self.options.members or [] + ret = [] + for name in memberlist: + if name in members: + ret.append(members[name]) + else: + logger.warning(__('missing attribute mentioned in :members: option: ' + 'module %s, attribute %s') % + (safe_getattr(self.object, '__name__', '???'), name), + type='autodoc') + return False, ret + + def sort_members(self, documenters: List[Tuple["Documenter", bool]], + order: str) -> List[Tuple["Documenter", bool]]: + if order == 'bysource' and self.__all__: + # Sort alphabetically first (for members not listed on the __all__) + documenters.sort(key=lambda e: e[0].name) + + # Sort by __all__ + def keyfunc(entry: Tuple[Documenter, bool]) -> int: + name = entry[0].name.split('::')[1] + if self.__all__ and name in self.__all__: + return self.__all__.index(name) + else: + return len(self.__all__) + documenters.sort(key=keyfunc) + + return documenters + else: + return super().sort_members(documenters, order) + + +class ModuleLevelDocumenter(Documenter): + """ + Specialized Documenter subclass for objects on module level (functions, + classes, data/constants). + """ + def resolve_name(self, modname: str, parents: Any, path: str, base: Any + ) -> Tuple[str, List[str]]: + if modname is None: + if path: + modname = path.rstrip('.') + else: + # if documenting a toplevel object without explicit module, + # it can be contained in another auto directive ... + modname = self.env.temp_data.get('autodoc:module') + # ... or in the scope of a module directive + if not modname: + modname = self.env.ref_context.get('py:module') + # ... else, it stays None, which means invalid + return modname, parents + [base] + + +class ClassLevelDocumenter(Documenter): + """ + Specialized Documenter subclass for objects on class level (methods, + attributes). + """ + def resolve_name(self, modname: str, parents: Any, path: str, base: Any + ) -> Tuple[str, List[str]]: + if modname is None: + if path: + mod_cls = path.rstrip('.') + else: + mod_cls = None + # if documenting a class-level object without path, + # there must be a current class, either from a parent + # auto directive ... + mod_cls = self.env.temp_data.get('autodoc:class') + # ... or from a class directive + if mod_cls is None: + mod_cls = self.env.ref_context.get('py:class') + # ... if still None, there's no way to know + if mod_cls is None: + return None, [] + modname, sep, cls = mod_cls.rpartition('.') + parents = [cls] + # if the module name is still missing, get it like above + if not modname: + modname = self.env.temp_data.get('autodoc:module') + if not modname: + modname = self.env.ref_context.get('py:module') + # ... else, it stays None, which means invalid + return modname, parents + [base] + + +class DocstringSignatureMixin: + """ + Mixin for FunctionDocumenter and MethodDocumenter to provide the + feature of reading the signature from the docstring. + """ + _new_docstrings: List[List[str]] = None + _signatures: List[str] = None + + def _find_signature(self) -> Tuple[str, str]: + # candidates of the object name + valid_names = [self.objpath[-1]] # type: ignore + if isinstance(self, ClassDocumenter): + valid_names.append('__init__') + if hasattr(self.object, '__mro__'): + valid_names.extend(cls.__name__ for cls in self.object.__mro__) + + docstrings = self.get_doc() + if docstrings is None: + return None, None + self._new_docstrings = docstrings[:] + self._signatures = [] + result = None + for i, doclines in enumerate(docstrings): + for j, line in enumerate(doclines): + if not line: + # no lines in docstring, no match + break + + if line.endswith('\\'): + line = line.rstrip('\\').rstrip() + + # match first line of docstring against signature RE + match = py_ext_sig_re.match(line) + if not match: + break + exmod, path, base, args, retann = match.groups() + + # the base name must match ours + if base not in valid_names: + break + + # re-prepare docstring to ignore more leading indentation + tab_width = self.directive.state.document.settings.tab_width # type: ignore + self._new_docstrings[i] = prepare_docstring('\n'.join(doclines[j + 1:]), + tab_width) + + if result is None: + # first signature + result = args, retann + else: + # subsequent signatures + self._signatures.append("(%s) -> %s" % (args, retann)) + + if result: + # finish the loop when signature found + break + + return result + + def get_doc(self) -> List[List[str]]: + if self._new_docstrings is not None: + return self._new_docstrings + return super().get_doc() # type: ignore + + def format_signature(self, **kwargs: Any) -> str: + if self.args is None and self.config.autodoc_docstring_signature: # type: ignore + # only act if a signature is not explicitly given already, and if + # the feature is enabled + result = self._find_signature() + if result is not None: + self.args, self.retann = result + sig = super().format_signature(**kwargs) # type: ignore + if self._signatures: + return "\n".join([sig] + self._signatures) + else: + return sig + + +class DocstringStripSignatureMixin(DocstringSignatureMixin): + """ + Mixin for AttributeDocumenter to provide the + feature of stripping any function signature from the docstring. + """ + def format_signature(self, **kwargs: Any) -> str: + if self.args is None and self.config.autodoc_docstring_signature: # type: ignore + # only act if a signature is not explicitly given already, and if + # the feature is enabled + result = self._find_signature() + if result is not None: + # Discarding _args is a only difference with + # DocstringSignatureMixin.format_signature. + # Documenter.format_signature use self.args value to format. + _args, self.retann = result + return super().format_signature(**kwargs) + + +class FunctionDocumenter(DocstringSignatureMixin, ModuleLevelDocumenter): # type: ignore + """ + Specialized Documenter subclass for functions. + """ + objtype = 'function' + member_order = 30 + + @classmethod + def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any + ) -> bool: + # supports functions, builtins and bound methods exported at the module level + return (inspect.isfunction(member) or inspect.isbuiltin(member) or + (inspect.isroutine(member) and isinstance(parent, ModuleDocumenter))) + + def format_args(self, **kwargs: Any) -> str: + if self.config.autodoc_typehints in ('none', 'description'): + kwargs.setdefault('show_annotation', False) + if self.config.autodoc_typehints_format == "short": + kwargs.setdefault('unqualified_typehints', True) + + try: + self.env.app.emit('autodoc-before-process-signature', self.object, False) + sig = inspect.signature(self.object, type_aliases=self.config.autodoc_type_aliases) + args = stringify_signature(sig, **kwargs) + except TypeError as exc: + logger.warning(__("Failed to get a function signature for %s: %s"), + self.fullname, exc) + return None + except ValueError: + args = '' + + if self.config.strip_signature_backslash: + # escape backslashes for reST + args = args.replace('\\', '\\\\') + return args + + def document_members(self, all_members: bool = False) -> None: + pass + + def add_directive_header(self, sig: str) -> None: + sourcename = self.get_sourcename() + super().add_directive_header(sig) + + if inspect.iscoroutinefunction(self.object) or inspect.isasyncgenfunction(self.object): + self.add_line(' :async:', sourcename) + + def format_signature(self, **kwargs: Any) -> str: + if self.config.autodoc_typehints_format == "short": + kwargs.setdefault('unqualified_typehints', True) + + sigs = [] + if (self.analyzer and + '.'.join(self.objpath) in self.analyzer.overloads and + self.config.autodoc_typehints != 'none'): + # Use signatures for overloaded functions instead of the implementation function. + overloaded = True + else: + overloaded = False + sig = super().format_signature(**kwargs) + sigs.append(sig) + + if inspect.is_singledispatch_function(self.object): + # append signature of singledispatch'ed functions + for typ, func in self.object.registry.items(): + if typ is object: + pass # default implementation. skipped. + else: + dispatchfunc = self.annotate_to_first_argument(func, typ) + if dispatchfunc: + documenter = FunctionDocumenter(self.directive, '') + documenter.object = dispatchfunc + documenter.objpath = [None] + sigs.append(documenter.format_signature()) + if overloaded: + actual = inspect.signature(self.object, + type_aliases=self.config.autodoc_type_aliases) + __globals__ = safe_getattr(self.object, '__globals__', {}) + for overload in self.analyzer.overloads.get('.'.join(self.objpath)): + overload = self.merge_default_value(actual, overload) + overload = evaluate_signature(overload, __globals__, + self.config.autodoc_type_aliases) + + sig = stringify_signature(overload, **kwargs) + sigs.append(sig) + + return "\n".join(sigs) + + def merge_default_value(self, actual: Signature, overload: Signature) -> Signature: + """Merge default values of actual implementation to the overload variants.""" + parameters = list(overload.parameters.values()) + for i, param in enumerate(parameters): + actual_param = actual.parameters.get(param.name) + if actual_param and param.default == '...': + parameters[i] = param.replace(default=actual_param.default) + + return overload.replace(parameters=parameters) + + def annotate_to_first_argument(self, func: Callable, typ: Type) -> Optional[Callable]: + """Annotate type hint to the first argument of function if needed.""" + try: + sig = inspect.signature(func, type_aliases=self.config.autodoc_type_aliases) + except TypeError as exc: + logger.warning(__("Failed to get a function signature for %s: %s"), + self.fullname, exc) + return None + except ValueError: + return None + + if len(sig.parameters) == 0: + return None + + def dummy(): + pass + + params = list(sig.parameters.values()) + if params[0].annotation is Parameter.empty: + params[0] = params[0].replace(annotation=typ) + try: + dummy.__signature__ = sig.replace(parameters=params) # type: ignore + return dummy + except (AttributeError, TypeError): + # failed to update signature (ex. built-in or extension types) + return None + + return func + + +class DecoratorDocumenter(FunctionDocumenter): + """ + Specialized Documenter subclass for decorator functions. + """ + objtype = 'decorator' + + # must be lower than FunctionDocumenter + priority = -1 + + def format_args(self, **kwargs: Any) -> Any: + args = super().format_args(**kwargs) + if ',' in args: + return args + else: + return None + + +# Types which have confusing metaclass signatures it would be best not to show. +# These are listed by name, rather than storing the objects themselves, to avoid +# needing to import the modules. +_METACLASS_CALL_BLACKLIST = [ + 'enum.EnumMeta.__call__', +] + + +# Types whose __new__ signature is a pass-through. +_CLASS_NEW_BLACKLIST = [ + 'typing.Generic.__new__', +] + + +class ClassDocumenter(DocstringSignatureMixin, ModuleLevelDocumenter): # type: ignore + """ + Specialized Documenter subclass for classes. + """ + objtype = 'class' + member_order = 20 + option_spec: OptionSpec = { + 'members': members_option, 'undoc-members': bool_option, + 'noindex': bool_option, 'inherited-members': inherited_members_option, + 'show-inheritance': bool_option, 'member-order': member_order_option, + 'exclude-members': exclude_members_option, + 'private-members': members_option, 'special-members': members_option, + 'class-doc-from': class_doc_from_option, + } + + _signature_class: Any = None + _signature_method_name: str = None + + def __init__(self, *args: Any) -> None: + super().__init__(*args) + + if self.config.autodoc_class_signature == 'separated': + self.options = self.options.copy() + + # show __init__() method + if self.options.special_members is None: + self.options['special-members'] = ['__new__', '__init__'] + else: + self.options.special_members.append('__new__') + self.options.special_members.append('__init__') + + merge_members_option(self.options) + + @classmethod + def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any + ) -> bool: + return isinstance(member, type) + + def import_object(self, raiseerror: bool = False) -> bool: + ret = super().import_object(raiseerror) + # if the class is documented under another name, document it + # as data/attribute + if ret: + if hasattr(self.object, '__name__'): + self.doc_as_attr = (self.objpath[-1] != self.object.__name__) + else: + self.doc_as_attr = True + return ret + + def _get_signature(self) -> Tuple[Optional[Any], Optional[str], Optional[Signature]]: + def get_user_defined_function_or_method(obj: Any, attr: str) -> Any: + """ Get the `attr` function or method from `obj`, if it is user-defined. """ + if inspect.is_builtin_class_method(obj, attr): + return None + attr = self.get_attr(obj, attr, None) + if not (inspect.ismethod(attr) or inspect.isfunction(attr)): + return None + return attr + + # This sequence is copied from inspect._signature_from_callable. + # ValueError means that no signature could be found, so we keep going. + + # First, we check the obj has a __signature__ attribute + if (hasattr(self.object, '__signature__') and + isinstance(self.object.__signature__, Signature)): + return None, None, self.object.__signature__ + + # Next, let's see if it has an overloaded __call__ defined + # in its metaclass + call = get_user_defined_function_or_method(type(self.object), '__call__') + + if call is not None: + if "{0.__module__}.{0.__qualname__}".format(call) in _METACLASS_CALL_BLACKLIST: + call = None + + if call is not None: + self.env.app.emit('autodoc-before-process-signature', call, True) + try: + sig = inspect.signature(call, bound_method=True, + type_aliases=self.config.autodoc_type_aliases) + return type(self.object), '__call__', sig + except ValueError: + pass + + # Now we check if the 'obj' class has a '__new__' method + new = get_user_defined_function_or_method(self.object, '__new__') + + if new is not None: + if "{0.__module__}.{0.__qualname__}".format(new) in _CLASS_NEW_BLACKLIST: + new = None + + if new is not None: + self.env.app.emit('autodoc-before-process-signature', new, True) + try: + sig = inspect.signature(new, bound_method=True, + type_aliases=self.config.autodoc_type_aliases) + return self.object, '__new__', sig + except ValueError: + pass + + # Finally, we should have at least __init__ implemented + init = get_user_defined_function_or_method(self.object, '__init__') + if init is not None: + self.env.app.emit('autodoc-before-process-signature', init, True) + try: + sig = inspect.signature(init, bound_method=True, + type_aliases=self.config.autodoc_type_aliases) + return self.object, '__init__', sig + except ValueError: + pass + + # None of the attributes are user-defined, so fall back to let inspect + # handle it. + # We don't know the exact method that inspect.signature will read + # the signature from, so just pass the object itself to our hook. + self.env.app.emit('autodoc-before-process-signature', self.object, False) + try: + sig = inspect.signature(self.object, bound_method=False, + type_aliases=self.config.autodoc_type_aliases) + return None, None, sig + except ValueError: + pass + + # Still no signature: happens e.g. for old-style classes + # with __init__ in C and no `__text_signature__`. + return None, None, None + + def format_args(self, **kwargs: Any) -> str: + if self.config.autodoc_typehints in ('none', 'description'): + kwargs.setdefault('show_annotation', False) + if self.config.autodoc_typehints_format == "short": + kwargs.setdefault('unqualified_typehints', True) + + try: + self._signature_class, self._signature_method_name, sig = self._get_signature() + except TypeError as exc: + # __signature__ attribute contained junk + logger.warning(__("Failed to get a constructor signature for %s: %s"), + self.fullname, exc) + return None + + if sig is None: + return None + + return stringify_signature(sig, show_return_annotation=False, **kwargs) + + def _find_signature(self) -> Tuple[str, str]: + result = super()._find_signature() + if result is not None: + # Strip a return value from signature of constructor in docstring (first entry) + result = (result[0], None) + + for i, sig in enumerate(self._signatures): + if sig.endswith(' -> None'): + # Strip a return value from signatures of constructor in docstring (subsequent + # entries) + self._signatures[i] = sig[:-8] + + return result + + def format_signature(self, **kwargs: Any) -> str: + if self.doc_as_attr: + return '' + if self.config.autodoc_class_signature == 'separated': + # do not show signatures + return '' + + if self.config.autodoc_typehints_format == "short": + kwargs.setdefault('unqualified_typehints', True) + + sig = super().format_signature() + sigs = [] + + overloads = self.get_overloaded_signatures() + if overloads and self.config.autodoc_typehints != 'none': + # Use signatures for overloaded methods instead of the implementation method. + method = safe_getattr(self._signature_class, self._signature_method_name, None) + __globals__ = safe_getattr(method, '__globals__', {}) + for overload in overloads: + overload = evaluate_signature(overload, __globals__, + self.config.autodoc_type_aliases) + + parameters = list(overload.parameters.values()) + overload = overload.replace(parameters=parameters[1:], + return_annotation=Parameter.empty) + sig = stringify_signature(overload, **kwargs) + sigs.append(sig) + else: + sigs.append(sig) + + return "\n".join(sigs) + + def get_overloaded_signatures(self) -> List[Signature]: + if self._signature_class and self._signature_method_name: + for cls in self._signature_class.__mro__: + try: + analyzer = ModuleAnalyzer.for_module(cls.__module__) + analyzer.analyze() + qualname = '.'.join([cls.__qualname__, self._signature_method_name]) + if qualname in analyzer.overloads: + return analyzer.overloads.get(qualname) + elif qualname in analyzer.tagorder: + # the constructor is defined in the class, but not overridden. + return [] + except PycodeError: + pass + + return [] + + def get_canonical_fullname(self) -> Optional[str]: + __modname__ = safe_getattr(self.object, '__module__', self.modname) + __qualname__ = safe_getattr(self.object, '__qualname__', None) + if __qualname__ is None: + __qualname__ = safe_getattr(self.object, '__name__', None) + if __qualname__ and '<locals>' in __qualname__: + # No valid qualname found if the object is defined as locals + __qualname__ = None + + if __modname__ and __qualname__: + return '.'.join([__modname__, __qualname__]) + else: + return None + + def add_directive_header(self, sig: str) -> None: + sourcename = self.get_sourcename() + + if self.doc_as_attr: + self.directivetype = 'attribute' + super().add_directive_header(sig) + + if self.analyzer and '.'.join(self.objpath) in self.analyzer.finals: + self.add_line(' :final:', sourcename) + + canonical_fullname = self.get_canonical_fullname() + if not self.doc_as_attr and canonical_fullname and self.fullname != canonical_fullname: + self.add_line(' :canonical: %s' % canonical_fullname, sourcename) + + # add inheritance info, if wanted + if not self.doc_as_attr and self.options.show_inheritance: + if inspect.getorigbases(self.object): + # A subclass of generic types + # refs: PEP-560 <https://peps.python.org/pep-0560/> + bases = list(self.object.__orig_bases__) + elif hasattr(self.object, '__bases__') and len(self.object.__bases__): + # A normal class + bases = list(self.object.__bases__) + else: + bases = [] + + self.env.events.emit('autodoc-process-bases', + self.fullname, self.object, self.options, bases) + + if self.config.autodoc_typehints_format == "short": + base_classes = [restify(cls, "smart") for cls in bases] + else: + base_classes = [restify(cls) for cls in bases] + + sourcename = self.get_sourcename() + self.add_line('', sourcename) + self.add_line(' ' + _('Bases: %s') % ', '.join(base_classes), sourcename) + + def get_object_members(self, want_all: bool) -> Tuple[bool, ObjectMembers]: + members = get_class_members(self.object, self.objpath, self.get_attr, + self.config.autodoc_inherit_docstrings) + if not want_all: + if not self.options.members: + return False, [] # type: ignore + # specific members given + selected = [] + for name in self.options.members: + if name in members: + selected.append(members[name]) + else: + logger.warning(__('missing attribute %s in object %s') % + (name, self.fullname), type='autodoc') + return False, selected + elif self.options.inherited_members: + return False, list(members.values()) + else: + return False, [m for m in members.values() if m.class_ == self.object] + + def get_doc(self) -> Optional[List[List[str]]]: + if self.doc_as_attr: + # Don't show the docstring of the class when it is an alias. + comment = self.get_variable_comment() + if comment: + return [] + else: + return None + + lines = getattr(self, '_new_docstrings', None) + if lines is not None: + return lines + + classdoc_from = self.options.get('class-doc-from', self.config.autoclass_content) + + docstrings = [] + attrdocstring = getdoc(self.object, self.get_attr) + if attrdocstring: + docstrings.append(attrdocstring) + + # for classes, what the "docstring" is can be controlled via a + # config value; the default is only the class docstring + if classdoc_from in ('both', 'init'): + __init__ = self.get_attr(self.object, '__init__', None) + initdocstring = getdoc(__init__, self.get_attr, + self.config.autodoc_inherit_docstrings, + self.object, '__init__') + # for new-style classes, no __init__ means default __init__ + if (initdocstring is not None and + (initdocstring == object.__init__.__doc__ or # for pypy + initdocstring.strip() == object.__init__.__doc__)): # for !pypy + initdocstring = None + if not initdocstring: + # try __new__ + __new__ = self.get_attr(self.object, '__new__', None) + initdocstring = getdoc(__new__, self.get_attr, + self.config.autodoc_inherit_docstrings, + self.object, '__new__') + # for new-style classes, no __new__ means default __new__ + if (initdocstring is not None and + (initdocstring == object.__new__.__doc__ or # for pypy + initdocstring.strip() == object.__new__.__doc__)): # for !pypy + initdocstring = None + if initdocstring: + if classdoc_from == 'init': + docstrings = [initdocstring] + else: + docstrings.append(initdocstring) + + tab_width = self.directive.state.document.settings.tab_width + return [prepare_docstring(docstring, tab_width) for docstring in docstrings] + + def get_variable_comment(self) -> Optional[List[str]]: + try: + key = ('', '.'.join(self.objpath)) + if self.doc_as_attr: + analyzer = ModuleAnalyzer.for_module(self.modname) + else: + analyzer = ModuleAnalyzer.for_module(self.get_real_modname()) + analyzer.analyze() + return list(analyzer.attr_docs.get(key, [])) + except PycodeError: + return None + + def add_content(self, more_content: Optional[StringList]) -> None: + if self.doc_as_attr and self.modname != self.get_real_modname(): + try: + # override analyzer to obtain doccomment around its definition. + self.analyzer = ModuleAnalyzer.for_module(self.modname) + self.analyzer.analyze() + except PycodeError: + pass + + if self.doc_as_attr and not self.get_variable_comment(): + try: + if self.config.autodoc_typehints_format == "short": + alias = restify(self.object, "smart") + else: + alias = restify(self.object) + more_content = StringList([_('alias of %s') % alias], source='') + except AttributeError: + pass # Invalid class object is passed. + + super().add_content(more_content) + + def document_members(self, all_members: bool = False) -> None: + if self.doc_as_attr: + return + super().document_members(all_members) + + def generate(self, more_content: Optional[StringList] = None, real_modname: str = None, + check_module: bool = False, all_members: bool = False) -> None: + # Do not pass real_modname and use the name from the __module__ + # attribute of the class. + # If a class gets imported into the module real_modname + # the analyzer won't find the source of the class, if + # it looks in real_modname. + return super().generate(more_content=more_content, + check_module=check_module, + all_members=all_members) + + +class ExceptionDocumenter(ClassDocumenter): + """ + Specialized ClassDocumenter subclass for exceptions. + """ + objtype = 'exception' + member_order = 10 + + # needs a higher priority than ClassDocumenter + priority = 10 + + @classmethod + def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any + ) -> bool: + return isinstance(member, type) and issubclass(member, BaseException) + + +class DataDocumenterMixinBase: + # define types of instance variables + config: Config = None + env: BuildEnvironment = None + modname: str = None + parent: Any = None + object: Any = None + objpath: List[str] = None + + def should_suppress_directive_header(self) -> bool: + """Check directive header should be suppressed.""" + return False + + def should_suppress_value_header(self) -> bool: + """Check :value: header should be suppressed.""" + return False + + def update_content(self, more_content: StringList) -> None: + """Update docstring for the NewType object.""" + pass + + +class GenericAliasMixin(DataDocumenterMixinBase): + """ + Mixin for DataDocumenter and AttributeDocumenter to provide the feature for + supporting GenericAliases. + """ + + def should_suppress_directive_header(self) -> bool: + return (inspect.isgenericalias(self.object) or + super().should_suppress_directive_header()) + + def update_content(self, more_content: StringList) -> None: + if inspect.isgenericalias(self.object): + if self.config.autodoc_typehints_format == "short": + alias = restify(self.object, "smart") + else: + alias = restify(self.object) + + more_content.append(_('alias of %s') % alias, '') + more_content.append('', '') + + super().update_content(more_content) + + +class NewTypeMixin(DataDocumenterMixinBase): + """ + Mixin for DataDocumenter and AttributeDocumenter to provide the feature for + supporting NewTypes. + """ + + def should_suppress_directive_header(self) -> bool: + return (inspect.isNewType(self.object) or + super().should_suppress_directive_header()) + + def update_content(self, more_content: StringList) -> None: + if inspect.isNewType(self.object): + if self.config.autodoc_typehints_format == "short": + supertype = restify(self.object.__supertype__, "smart") + else: + supertype = restify(self.object.__supertype__) + + more_content.append(_('alias of %s') % supertype, '') + more_content.append('', '') + + super().update_content(more_content) + + +class TypeVarMixin(DataDocumenterMixinBase): + """ + Mixin for DataDocumenter and AttributeDocumenter to provide the feature for + supporting TypeVars. + """ + + def should_suppress_directive_header(self) -> bool: + return (isinstance(self.object, TypeVar) or + super().should_suppress_directive_header()) + + def get_doc(self) -> Optional[List[List[str]]]: + if isinstance(self.object, TypeVar): + if self.object.__doc__ != TypeVar.__doc__: + return super().get_doc() # type: ignore + else: + return [] + else: + return super().get_doc() # type: ignore + + def update_content(self, more_content: StringList) -> None: + if isinstance(self.object, TypeVar): + attrs = [repr(self.object.__name__)] + for constraint in self.object.__constraints__: + if self.config.autodoc_typehints_format == "short": + attrs.append(stringify_typehint(constraint, "smart")) + else: + attrs.append(stringify_typehint(constraint)) + if self.object.__bound__: + if self.config.autodoc_typehints_format == "short": + bound = restify(self.object.__bound__, "smart") + else: + bound = restify(self.object.__bound__) + attrs.append(r"bound=\ " + bound) + if self.object.__covariant__: + attrs.append("covariant=True") + if self.object.__contravariant__: + attrs.append("contravariant=True") + + more_content.append(_('alias of TypeVar(%s)') % ", ".join(attrs), '') + more_content.append('', '') + + super().update_content(more_content) + + +class UninitializedGlobalVariableMixin(DataDocumenterMixinBase): + """ + Mixin for DataDocumenter to provide the feature for supporting uninitialized + (type annotation only) global variables. + """ + + def import_object(self, raiseerror: bool = False) -> bool: + try: + return super().import_object(raiseerror=True) # type: ignore + except ImportError as exc: + # annotation only instance variable (PEP-526) + try: + with mock(self.config.autodoc_mock_imports): + parent = import_module(self.modname, self.config.autodoc_warningiserror) + annotations = get_type_hints(parent, None, + self.config.autodoc_type_aliases) + if self.objpath[-1] in annotations: + self.object = UNINITIALIZED_ATTR + self.parent = parent + return True + except ImportError: + pass + + if raiseerror: + raise + else: + logger.warning(exc.args[0], type='autodoc', subtype='import_object') + self.env.note_reread() + return False + + def should_suppress_value_header(self) -> bool: + return (self.object is UNINITIALIZED_ATTR or + super().should_suppress_value_header()) + + def get_doc(self) -> Optional[List[List[str]]]: + if self.object is UNINITIALIZED_ATTR: + return [] + else: + return super().get_doc() # type: ignore + + +class DataDocumenter(GenericAliasMixin, NewTypeMixin, TypeVarMixin, + UninitializedGlobalVariableMixin, ModuleLevelDocumenter): + """ + Specialized Documenter subclass for data items. + """ + objtype = 'data' + member_order = 40 + priority = -10 + option_spec: OptionSpec = dict(ModuleLevelDocumenter.option_spec) + option_spec["annotation"] = annotation_option + option_spec["no-value"] = bool_option + + @classmethod + def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any + ) -> bool: + return isinstance(parent, ModuleDocumenter) and isattr + + def update_annotations(self, parent: Any) -> None: + """Update __annotations__ to support type_comment and so on.""" + annotations = dict(inspect.getannotations(parent)) + parent.__annotations__ = annotations + + try: + analyzer = ModuleAnalyzer.for_module(self.modname) + analyzer.analyze() + for (classname, attrname), annotation in analyzer.annotations.items(): + if classname == '' and attrname not in annotations: + annotations[attrname] = annotation + except PycodeError: + pass + + def import_object(self, raiseerror: bool = False) -> bool: + ret = super().import_object(raiseerror) + if self.parent: + self.update_annotations(self.parent) + + return ret + + def should_suppress_value_header(self) -> bool: + if super().should_suppress_value_header(): + return True + else: + doc = self.get_doc() + docstring, metadata = separate_metadata('\n'.join(sum(doc, []))) + if 'hide-value' in metadata: + return True + + return False + + def add_directive_header(self, sig: str) -> None: + super().add_directive_header(sig) + sourcename = self.get_sourcename() + if self.options.annotation is SUPPRESS or self.should_suppress_directive_header(): + pass + elif self.options.annotation: + self.add_line(' :annotation: %s' % self.options.annotation, + sourcename) + else: + if self.config.autodoc_typehints != 'none': + # obtain annotation for this data + annotations = get_type_hints(self.parent, None, + self.config.autodoc_type_aliases) + if self.objpath[-1] in annotations: + if self.config.autodoc_typehints_format == "short": + objrepr = stringify_typehint(annotations.get(self.objpath[-1]), + "smart") + else: + objrepr = stringify_typehint(annotations.get(self.objpath[-1])) + self.add_line(' :type: ' + objrepr, sourcename) + + try: + if (self.options.no_value or self.should_suppress_value_header() or + ismock(self.object)): + pass + else: + objrepr = object_description(self.object) + self.add_line(' :value: ' + objrepr, sourcename) + except ValueError: + pass + + def document_members(self, all_members: bool = False) -> None: + pass + + def get_real_modname(self) -> str: + real_modname = self.get_attr(self.parent or self.object, '__module__', None) + return real_modname or self.modname + + def get_module_comment(self, attrname: str) -> Optional[List[str]]: + try: + analyzer = ModuleAnalyzer.for_module(self.modname) + analyzer.analyze() + key = ('', attrname) + if key in analyzer.attr_docs: + return list(analyzer.attr_docs[key]) + except PycodeError: + pass + + return None + + def get_doc(self) -> Optional[List[List[str]]]: + # Check the variable has a docstring-comment + comment = self.get_module_comment(self.objpath[-1]) + if comment: + return [comment] + else: + return super().get_doc() + + def add_content(self, more_content: Optional[StringList]) -> None: + # Disable analyzing variable comment on Documenter.add_content() to control it on + # DataDocumenter.add_content() + self.analyzer = None + + if not more_content: + more_content = StringList() + + self.update_content(more_content) + super().add_content(more_content) + + +class NewTypeDataDocumenter(DataDocumenter): + """ + Specialized Documenter subclass for NewTypes. + + Note: This must be invoked before FunctionDocumenter because NewType is a kind of + function object. + """ + + objtype = 'newtypedata' + directivetype = 'data' + priority = FunctionDocumenter.priority + 1 + + @classmethod + def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any + ) -> bool: + return inspect.isNewType(member) and isattr + + +class MethodDocumenter(DocstringSignatureMixin, ClassLevelDocumenter): # type: ignore + """ + Specialized Documenter subclass for methods (normal, static and class). + """ + objtype = 'method' + directivetype = 'method' + member_order = 50 + priority = 1 # must be more than FunctionDocumenter + + @classmethod + def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any + ) -> bool: + return inspect.isroutine(member) and not isinstance(parent, ModuleDocumenter) + + def import_object(self, raiseerror: bool = False) -> bool: + ret = super().import_object(raiseerror) + if not ret: + return ret + + # to distinguish classmethod/staticmethod + obj = self.parent.__dict__.get(self.object_name) + if obj is None: + obj = self.object + + if (inspect.isclassmethod(obj) or + inspect.isstaticmethod(obj, cls=self.parent, name=self.object_name)): + # document class and static members before ordinary ones + self.member_order = self.member_order - 1 + + return ret + + def format_args(self, **kwargs: Any) -> str: + if self.config.autodoc_typehints in ('none', 'description'): + kwargs.setdefault('show_annotation', False) + if self.config.autodoc_typehints_format == "short": + kwargs.setdefault('unqualified_typehints', True) + + try: + if self.object == object.__init__ and self.parent != object: + # Classes not having own __init__() method are shown as no arguments. + # + # Note: The signature of object.__init__() is (self, /, *args, **kwargs). + # But it makes users confused. + args = '()' + else: + if inspect.isstaticmethod(self.object, cls=self.parent, name=self.object_name): + self.env.app.emit('autodoc-before-process-signature', self.object, False) + sig = inspect.signature(self.object, bound_method=False, + type_aliases=self.config.autodoc_type_aliases) + else: + self.env.app.emit('autodoc-before-process-signature', self.object, True) + sig = inspect.signature(self.object, bound_method=True, + type_aliases=self.config.autodoc_type_aliases) + args = stringify_signature(sig, **kwargs) + except TypeError as exc: + logger.warning(__("Failed to get a method signature for %s: %s"), + self.fullname, exc) + return None + except ValueError: + args = '' + + if self.config.strip_signature_backslash: + # escape backslashes for reST + args = args.replace('\\', '\\\\') + return args + + def add_directive_header(self, sig: str) -> None: + super().add_directive_header(sig) + + sourcename = self.get_sourcename() + obj = self.parent.__dict__.get(self.object_name, self.object) + if inspect.isabstractmethod(obj): + self.add_line(' :abstractmethod:', sourcename) + if inspect.iscoroutinefunction(obj) or inspect.isasyncgenfunction(obj): + self.add_line(' :async:', sourcename) + if inspect.isclassmethod(obj): + self.add_line(' :classmethod:', sourcename) + if inspect.isstaticmethod(obj, cls=self.parent, name=self.object_name): + self.add_line(' :staticmethod:', sourcename) + if self.analyzer and '.'.join(self.objpath) in self.analyzer.finals: + self.add_line(' :final:', sourcename) + + def document_members(self, all_members: bool = False) -> None: + pass + + def format_signature(self, **kwargs: Any) -> str: + if self.config.autodoc_typehints_format == "short": + kwargs.setdefault('unqualified_typehints', True) + + sigs = [] + if (self.analyzer and + '.'.join(self.objpath) in self.analyzer.overloads and + self.config.autodoc_typehints != 'none'): + # Use signatures for overloaded methods instead of the implementation method. + overloaded = True + else: + overloaded = False + sig = super().format_signature(**kwargs) + sigs.append(sig) + + meth = self.parent.__dict__.get(self.objpath[-1]) + if inspect.is_singledispatch_method(meth): + # append signature of singledispatch'ed functions + for typ, func in meth.dispatcher.registry.items(): + if typ is object: + pass # default implementation. skipped. + else: + dispatchmeth = self.annotate_to_first_argument(func, typ) + if dispatchmeth: + documenter = MethodDocumenter(self.directive, '') + documenter.parent = self.parent + documenter.object = dispatchmeth + documenter.objpath = [None] + sigs.append(documenter.format_signature()) + if overloaded: + if inspect.isstaticmethod(self.object, cls=self.parent, name=self.object_name): + actual = inspect.signature(self.object, bound_method=False, + type_aliases=self.config.autodoc_type_aliases) + else: + actual = inspect.signature(self.object, bound_method=True, + type_aliases=self.config.autodoc_type_aliases) + + __globals__ = safe_getattr(self.object, '__globals__', {}) + for overload in self.analyzer.overloads.get('.'.join(self.objpath)): + overload = self.merge_default_value(actual, overload) + overload = evaluate_signature(overload, __globals__, + self.config.autodoc_type_aliases) + + if not inspect.isstaticmethod(self.object, cls=self.parent, + name=self.object_name): + parameters = list(overload.parameters.values()) + overload = overload.replace(parameters=parameters[1:]) + sig = stringify_signature(overload, **kwargs) + sigs.append(sig) + + return "\n".join(sigs) + + def merge_default_value(self, actual: Signature, overload: Signature) -> Signature: + """Merge default values of actual implementation to the overload variants.""" + parameters = list(overload.parameters.values()) + for i, param in enumerate(parameters): + actual_param = actual.parameters.get(param.name) + if actual_param and param.default == '...': + parameters[i] = param.replace(default=actual_param.default) + + return overload.replace(parameters=parameters) + + def annotate_to_first_argument(self, func: Callable, typ: Type) -> Optional[Callable]: + """Annotate type hint to the first argument of function if needed.""" + try: + sig = inspect.signature(func, type_aliases=self.config.autodoc_type_aliases) + except TypeError as exc: + logger.warning(__("Failed to get a method signature for %s: %s"), + self.fullname, exc) + return None + except ValueError: + return None + + if len(sig.parameters) == 1: + return None + + def dummy(): + pass + + params = list(sig.parameters.values()) + if params[1].annotation is Parameter.empty: + params[1] = params[1].replace(annotation=typ) + try: + dummy.__signature__ = sig.replace(parameters=params) # type: ignore + return dummy + except (AttributeError, TypeError): + # failed to update signature (ex. built-in or extension types) + return None + + return func + + def get_doc(self) -> Optional[List[List[str]]]: + if self._new_docstrings is not None: + # docstring already returned previously, then modified by + # `DocstringSignatureMixin`. Just return the previously-computed + # result, so that we don't lose the processing done by + # `DocstringSignatureMixin`. + return self._new_docstrings + if self.objpath[-1] == '__init__': + docstring = getdoc(self.object, self.get_attr, + self.config.autodoc_inherit_docstrings, + self.parent, self.object_name) + if (docstring is not None and + (docstring == object.__init__.__doc__ or # for pypy + docstring.strip() == object.__init__.__doc__)): # for !pypy + docstring = None + if docstring: + tab_width = self.directive.state.document.settings.tab_width + return [prepare_docstring(docstring, tabsize=tab_width)] + else: + return [] + elif self.objpath[-1] == '__new__': + docstring = getdoc(self.object, self.get_attr, + self.config.autodoc_inherit_docstrings, + self.parent, self.object_name) + if (docstring is not None and + (docstring == object.__new__.__doc__ or # for pypy + docstring.strip() == object.__new__.__doc__)): # for !pypy + docstring = None + if docstring: + tab_width = self.directive.state.document.settings.tab_width + return [prepare_docstring(docstring, tabsize=tab_width)] + else: + return [] + else: + return super().get_doc() + + +class NonDataDescriptorMixin(DataDocumenterMixinBase): + """ + Mixin for AttributeDocumenter to provide the feature for supporting non + data-descriptors. + + .. note:: This mix-in must be inherited after other mix-ins. Otherwise, docstring + and :value: header will be suppressed unexpectedly. + """ + + def import_object(self, raiseerror: bool = False) -> bool: + ret = super().import_object(raiseerror) # type: ignore + if ret and not inspect.isattributedescriptor(self.object): + self.non_data_descriptor = True + else: + self.non_data_descriptor = False + + return ret + + def should_suppress_value_header(self) -> bool: + return (not getattr(self, 'non_data_descriptor', False) or + super().should_suppress_directive_header()) + + def get_doc(self) -> Optional[List[List[str]]]: + if getattr(self, 'non_data_descriptor', False): + # the docstring of non datadescriptor is very probably the wrong thing + # to display + return None + else: + return super().get_doc() # type: ignore + + +class SlotsMixin(DataDocumenterMixinBase): + """ + Mixin for AttributeDocumenter to provide the feature for supporting __slots__. + """ + + def isslotsattribute(self) -> bool: + """Check the subject is an attribute in __slots__.""" + try: + __slots__ = inspect.getslots(self.parent) + if __slots__ and self.objpath[-1] in __slots__: + return True + else: + return False + except (ValueError, TypeError): + return False + + def import_object(self, raiseerror: bool = False) -> bool: + ret = super().import_object(raiseerror) # type: ignore + if self.isslotsattribute(): + self.object = SLOTSATTR + + return ret + + def should_suppress_value_header(self) -> bool: + if self.object is SLOTSATTR: + return True + else: + return super().should_suppress_value_header() + + def get_doc(self) -> Optional[List[List[str]]]: + if self.object is SLOTSATTR: + try: + __slots__ = inspect.getslots(self.parent) + if __slots__ and __slots__.get(self.objpath[-1]): + docstring = prepare_docstring(__slots__[self.objpath[-1]]) + return [docstring] + else: + return [] + except ValueError as exc: + logger.warning(__('Invalid __slots__ found on %s. Ignored.'), + (self.parent.__qualname__, exc), type='autodoc') + return [] + else: + return super().get_doc() # type: ignore + + @property + def _datadescriptor(self) -> bool: + warnings.warn('AttributeDocumenter._datadescriptor() is deprecated.', + RemovedInSphinx60Warning) + if self.object is SLOTSATTR: + return True + else: + return False + + +class RuntimeInstanceAttributeMixin(DataDocumenterMixinBase): + """ + Mixin for AttributeDocumenter to provide the feature for supporting runtime + instance attributes (that are defined in __init__() methods with doc-comments). + + Example: + + class Foo: + def __init__(self): + self.attr = None #: This is a target of this mix-in. + """ + + RUNTIME_INSTANCE_ATTRIBUTE = object() + + def is_runtime_instance_attribute(self, parent: Any) -> bool: + """Check the subject is an attribute defined in __init__().""" + # An instance variable defined in __init__(). + if self.get_attribute_comment(parent, self.objpath[-1]): # type: ignore + return True + elif self.is_runtime_instance_attribute_not_commented(parent): + return True + else: + return False + + def is_runtime_instance_attribute_not_commented(self, parent: Any) -> bool: + """Check the subject is an attribute defined in __init__() without comment.""" + for cls in inspect.getmro(parent): + try: + module = safe_getattr(cls, '__module__') + qualname = safe_getattr(cls, '__qualname__') + + analyzer = ModuleAnalyzer.for_module(module) + analyzer.analyze() + if qualname and self.objpath: + key = '.'.join([qualname, self.objpath[-1]]) + if key in analyzer.tagorder: + return True + except (AttributeError, PycodeError): + pass + + return None + + def import_object(self, raiseerror: bool = False) -> bool: + """Check the existence of runtime instance attribute after failing to import the + attribute.""" + try: + return super().import_object(raiseerror=True) # type: ignore + except ImportError as exc: + try: + with mock(self.config.autodoc_mock_imports): + ret = import_object(self.modname, self.objpath[:-1], 'class', + attrgetter=self.get_attr, # type: ignore + warningiserror=self.config.autodoc_warningiserror) + parent = ret[3] + if self.is_runtime_instance_attribute(parent): + self.object = self.RUNTIME_INSTANCE_ATTRIBUTE + self.parent = parent + return True + except ImportError: + pass + + if raiseerror: + raise + else: + logger.warning(exc.args[0], type='autodoc', subtype='import_object') + self.env.note_reread() + return False + + def should_suppress_value_header(self) -> bool: + return (self.object is self.RUNTIME_INSTANCE_ATTRIBUTE or + super().should_suppress_value_header()) + + def get_doc(self) -> Optional[List[List[str]]]: + if (self.object is self.RUNTIME_INSTANCE_ATTRIBUTE and + self.is_runtime_instance_attribute_not_commented(self.parent)): + return None + else: + return super().get_doc() # type: ignore + + +class UninitializedInstanceAttributeMixin(DataDocumenterMixinBase): + """ + Mixin for AttributeDocumenter to provide the feature for supporting uninitialized + instance attributes (PEP-526 styled, annotation only attributes). + + Example: + + class Foo: + attr: int #: This is a target of this mix-in. + """ + + def is_uninitialized_instance_attribute(self, parent: Any) -> bool: + """Check the subject is an annotation only attribute.""" + annotations = get_type_hints(parent, None, self.config.autodoc_type_aliases) + if self.objpath[-1] in annotations: + return True + else: + return False + + def import_object(self, raiseerror: bool = False) -> bool: + """Check the exisitence of uninitialized instance attribute when failed to import + the attribute.""" + try: + return super().import_object(raiseerror=True) # type: ignore + except ImportError as exc: + try: + ret = import_object(self.modname, self.objpath[:-1], 'class', + attrgetter=self.get_attr, # type: ignore + warningiserror=self.config.autodoc_warningiserror) + parent = ret[3] + if self.is_uninitialized_instance_attribute(parent): + self.object = UNINITIALIZED_ATTR + self.parent = parent + return True + except ImportError: + pass + + if raiseerror: + raise + else: + logger.warning(exc.args[0], type='autodoc', subtype='import_object') + self.env.note_reread() + return False + + def should_suppress_value_header(self) -> bool: + return (self.object is UNINITIALIZED_ATTR or + super().should_suppress_value_header()) + + def get_doc(self) -> Optional[List[List[str]]]: + if self.object is UNINITIALIZED_ATTR: + return None + else: + return super().get_doc() # type: ignore + + +class AttributeDocumenter(GenericAliasMixin, NewTypeMixin, SlotsMixin, # type: ignore + TypeVarMixin, RuntimeInstanceAttributeMixin, + UninitializedInstanceAttributeMixin, NonDataDescriptorMixin, + DocstringStripSignatureMixin, ClassLevelDocumenter): + """ + Specialized Documenter subclass for attributes. + """ + objtype = 'attribute' + member_order = 60 + option_spec: OptionSpec = dict(ModuleLevelDocumenter.option_spec) + option_spec["annotation"] = annotation_option + option_spec["no-value"] = bool_option + + # must be higher than the MethodDocumenter, else it will recognize + # some non-data descriptors as methods + priority = 10 + + @staticmethod + def is_function_or_method(obj: Any) -> bool: + return inspect.isfunction(obj) or inspect.isbuiltin(obj) or inspect.ismethod(obj) + + @classmethod + def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any + ) -> bool: + if isinstance(parent, ModuleDocumenter): + return False + elif inspect.isattributedescriptor(member): + return True + elif not inspect.isroutine(member) and not isinstance(member, type): + return True + else: + return False + + def document_members(self, all_members: bool = False) -> None: + pass + + def update_annotations(self, parent: Any) -> None: + """Update __annotations__ to support type_comment and so on.""" + try: + annotations = dict(inspect.getannotations(parent)) + parent.__annotations__ = annotations + + for cls in inspect.getmro(parent): + try: + module = safe_getattr(cls, '__module__') + qualname = safe_getattr(cls, '__qualname__') + + analyzer = ModuleAnalyzer.for_module(module) + analyzer.analyze() + for (classname, attrname), annotation in analyzer.annotations.items(): + if classname == qualname and attrname not in annotations: + annotations[attrname] = annotation + except (AttributeError, PycodeError): + pass + except (AttributeError, TypeError): + # Failed to set __annotations__ (built-in, extensions, etc.) + pass + + def import_object(self, raiseerror: bool = False) -> bool: + ret = super().import_object(raiseerror) + if inspect.isenumattribute(self.object): + self.object = self.object.value + if self.parent: + self.update_annotations(self.parent) + + return ret + + def get_real_modname(self) -> str: + real_modname = self.get_attr(self.parent or self.object, '__module__', None) + return real_modname or self.modname + + def should_suppress_value_header(self) -> bool: + if super().should_suppress_value_header(): + return True + else: + doc = self.get_doc() + if doc: + docstring, metadata = separate_metadata('\n'.join(sum(doc, []))) + if 'hide-value' in metadata: + return True + + return False + + def add_directive_header(self, sig: str) -> None: + super().add_directive_header(sig) + sourcename = self.get_sourcename() + if self.options.annotation is SUPPRESS or self.should_suppress_directive_header(): + pass + elif self.options.annotation: + self.add_line(' :annotation: %s' % self.options.annotation, sourcename) + else: + if self.config.autodoc_typehints != 'none': + # obtain type annotation for this attribute + annotations = get_type_hints(self.parent, None, + self.config.autodoc_type_aliases) + if self.objpath[-1] in annotations: + if self.config.autodoc_typehints_format == "short": + objrepr = stringify_typehint(annotations.get(self.objpath[-1]), + "smart") + else: + objrepr = stringify_typehint(annotations.get(self.objpath[-1])) + self.add_line(' :type: ' + objrepr, sourcename) + + try: + if (self.options.no_value or self.should_suppress_value_header() or + ismock(self.object)): + pass + else: + objrepr = object_description(self.object) + self.add_line(' :value: ' + objrepr, sourcename) + except ValueError: + pass + + def get_attribute_comment(self, parent: Any, attrname: str) -> Optional[List[str]]: + for cls in inspect.getmro(parent): + try: + module = safe_getattr(cls, '__module__') + qualname = safe_getattr(cls, '__qualname__') + + analyzer = ModuleAnalyzer.for_module(module) + analyzer.analyze() + if qualname and self.objpath: + key = (qualname, attrname) + if key in analyzer.attr_docs: + return list(analyzer.attr_docs[key]) + except (AttributeError, PycodeError): + pass + + return None + + def get_doc(self) -> Optional[List[List[str]]]: + # Check the attribute has a docstring-comment + comment = self.get_attribute_comment(self.parent, self.objpath[-1]) + if comment: + return [comment] + + try: + # Disable `autodoc_inherit_docstring` temporarily to avoid to obtain + # a docstring from the value which descriptor returns unexpectedly. + # ref: https://github.com/sphinx-doc/sphinx/issues/7805 + orig = self.config.autodoc_inherit_docstrings + self.config.autodoc_inherit_docstrings = False # type: ignore + return super().get_doc() + finally: + self.config.autodoc_inherit_docstrings = orig # type: ignore + + def add_content(self, more_content: Optional[StringList]) -> None: + # Disable analyzing attribute comment on Documenter.add_content() to control it on + # AttributeDocumenter.add_content() + self.analyzer = None + + if more_content is None: + more_content = StringList() + self.update_content(more_content) + super().add_content(more_content) + + +class PropertyDocumenter(DocstringStripSignatureMixin, ClassLevelDocumenter): # type: ignore + """ + Specialized Documenter subclass for properties. + """ + objtype = 'property' + member_order = 60 + + # before AttributeDocumenter + priority = AttributeDocumenter.priority + 1 + + @classmethod + def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any + ) -> bool: + if isinstance(parent, ClassDocumenter): + if inspect.isproperty(member): + return True + else: + __dict__ = safe_getattr(parent.object, '__dict__', {}) + obj = __dict__.get(membername) + return isinstance(obj, classmethod) and inspect.isproperty(obj.__func__) + else: + return False + + def import_object(self, raiseerror: bool = False) -> bool: + """Check the exisitence of uninitialized instance attribute when failed to import + the attribute.""" + ret = super().import_object(raiseerror) + if ret and not inspect.isproperty(self.object): + __dict__ = safe_getattr(self.parent, '__dict__', {}) + obj = __dict__.get(self.objpath[-1]) + if isinstance(obj, classmethod) and inspect.isproperty(obj.__func__): + self.object = obj.__func__ + self.isclassmethod = True + return True + else: + return False + + self.isclassmethod = False + return ret + + def document_members(self, all_members: bool = False) -> None: + pass + + def get_real_modname(self) -> str: + real_modname = self.get_attr(self.parent or self.object, '__module__', None) + return real_modname or self.modname + + def add_directive_header(self, sig: str) -> None: + super().add_directive_header(sig) + sourcename = self.get_sourcename() + if inspect.isabstractmethod(self.object): + self.add_line(' :abstractmethod:', sourcename) + if self.isclassmethod: + self.add_line(' :classmethod:', sourcename) + + if safe_getattr(self.object, 'fget', None): # property + func = self.object.fget + elif safe_getattr(self.object, 'func', None): # cached_property + func = self.object.func + else: + func = None + + if func and self.config.autodoc_typehints != 'none': + try: + signature = inspect.signature(func, + type_aliases=self.config.autodoc_type_aliases) + if signature.return_annotation is not Parameter.empty: + if self.config.autodoc_typehints_format == "short": + objrepr = stringify_typehint(signature.return_annotation, "smart") + else: + objrepr = stringify_typehint(signature.return_annotation) + self.add_line(' :type: ' + objrepr, sourcename) + except TypeError as exc: + logger.warning(__("Failed to get a function signature for %s: %s"), + self.fullname, exc) + return None + except ValueError: + return None + + +class NewTypeAttributeDocumenter(AttributeDocumenter): + """ + Specialized Documenter subclass for NewTypes. + + Note: This must be invoked before MethodDocumenter because NewType is a kind of + function object. + """ + + objtype = 'newvarattribute' + directivetype = 'attribute' + priority = MethodDocumenter.priority + 1 + + @classmethod + def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any + ) -> bool: + return not isinstance(parent, ModuleDocumenter) and inspect.isNewType(member) + + +def autodoc_attrgetter(app: Sphinx, obj: Any, name: str, *defargs: Any) -> Any: + """Alternative getattr() for types""" + for typ, func in app.registry.autodoc_attrgettrs.items(): + if isinstance(obj, typ): + return func(obj, name, *defargs) + + return safe_getattr(obj, name, *defargs) + + +def setup(app: Sphinx) -> Dict[str, Any]: + app.add_autodocumenter(ModuleDocumenter) + app.add_autodocumenter(ClassDocumenter) + app.add_autodocumenter(ExceptionDocumenter) + app.add_autodocumenter(DataDocumenter) + app.add_autodocumenter(NewTypeDataDocumenter) + app.add_autodocumenter(FunctionDocumenter) + app.add_autodocumenter(DecoratorDocumenter) + app.add_autodocumenter(MethodDocumenter) + app.add_autodocumenter(AttributeDocumenter) + app.add_autodocumenter(PropertyDocumenter) + app.add_autodocumenter(NewTypeAttributeDocumenter) + + app.add_config_value('autoclass_content', 'class', True, ENUM('both', 'class', 'init')) + app.add_config_value('autodoc_member_order', 'alphabetical', True, + ENUM('alphabetical', 'bysource', 'groupwise')) + app.add_config_value('autodoc_class_signature', 'mixed', True, ENUM('mixed', 'separated')) + app.add_config_value('autodoc_default_options', {}, True) + app.add_config_value('autodoc_docstring_signature', True, True) + app.add_config_value('autodoc_mock_imports', [], True) + app.add_config_value('autodoc_typehints', "signature", True, + ENUM("signature", "description", "none", "both")) + app.add_config_value('autodoc_typehints_description_target', 'all', True, + ENUM('all', 'documented', 'documented_params')) + app.add_config_value('autodoc_type_aliases', {}, True) + app.add_config_value('autodoc_typehints_format', "short", 'env', + ENUM("fully-qualified", "short")) + app.add_config_value('autodoc_warningiserror', True, True) + app.add_config_value('autodoc_inherit_docstrings', True, True) + app.add_event('autodoc-before-process-signature') + app.add_event('autodoc-process-docstring') + app.add_event('autodoc-process-signature') + app.add_event('autodoc-skip-member') + app.add_event('autodoc-process-bases') + + app.setup_extension('sphinx.ext.autodoc.preserve_defaults') + app.setup_extension('sphinx.ext.autodoc.type_comment') + app.setup_extension('sphinx.ext.autodoc.typehints') + + return {'version': sphinx.__display_version__, 'parallel_read_safe': True} diff --git a/sphinx/ext/autodoc/directive.py b/sphinx/ext/autodoc/directive.py new file mode 100644 index 0000000..f210a7d --- /dev/null +++ b/sphinx/ext/autodoc/directive.py @@ -0,0 +1,160 @@ +import warnings +from typing import Any, Callable, Dict, List, Optional, Set, Type + +from docutils import nodes +from docutils.nodes import Element, Node +from docutils.parsers.rst.states import RSTState +from docutils.statemachine import StringList +from docutils.utils import Reporter, assemble_option_dict + +from sphinx.config import Config +from sphinx.deprecation import RemovedInSphinx60Warning +from sphinx.environment import BuildEnvironment +from sphinx.ext.autodoc import Documenter, Options +from sphinx.util import logging +from sphinx.util.docutils import SphinxDirective, switch_source_input +from sphinx.util.nodes import nested_parse_with_titles + +logger = logging.getLogger(__name__) + + +# common option names for autodoc directives +AUTODOC_DEFAULT_OPTIONS = ['members', 'undoc-members', 'inherited-members', + 'show-inheritance', 'private-members', 'special-members', + 'ignore-module-all', 'exclude-members', 'member-order', + 'imported-members', 'class-doc-from', 'no-value'] + +AUTODOC_EXTENDABLE_OPTIONS = ['members', 'private-members', 'special-members', + 'exclude-members'] + + +class DummyOptionSpec(dict): + """An option_spec allows any options.""" + + def __bool__(self) -> bool: + """Behaves like some options are defined.""" + return True + + def __getitem__(self, key: str) -> Callable[[str], str]: + return lambda x: x + + +class DocumenterBridge: + """A parameters container for Documenters.""" + + def __init__(self, env: BuildEnvironment, reporter: Optional[Reporter], options: Options, + lineno: int, state: Any) -> None: + self.env = env + self._reporter = reporter + self.genopt = options + self.lineno = lineno + self.record_dependencies: Set[str] = set() + self.result = StringList() + self.state = state + + def warn(self, msg: str) -> None: + warnings.warn('DocumenterBridge.warn is deprecated. Please use sphinx.util.logging ' + 'module instead.', + RemovedInSphinx60Warning, stacklevel=2) + logger.warning(msg, location=(self.env.docname, self.lineno)) + + @property + def filename_set(self) -> Set: + warnings.warn('DocumenterBridge.filename_set is deprecated.', + RemovedInSphinx60Warning, stacklevel=2) + return self.record_dependencies + + +def process_documenter_options(documenter: Type[Documenter], config: Config, options: Dict + ) -> Options: + """Recognize options of Documenter from user input.""" + for name in AUTODOC_DEFAULT_OPTIONS: + if name not in documenter.option_spec: + continue + else: + negated = options.pop('no-' + name, True) is None + if name in config.autodoc_default_options and not negated: + if name in options and isinstance(config.autodoc_default_options[name], str): + # take value from options if present or extend it + # with autodoc_default_options if necessary + if name in AUTODOC_EXTENDABLE_OPTIONS: + if options[name] is not None and options[name].startswith('+'): + options[name] = ','.join([config.autodoc_default_options[name], + options[name][1:]]) + else: + options[name] = config.autodoc_default_options[name] + + elif options.get(name) is not None: + # remove '+' from option argument if there's nothing to merge it with + options[name] = options[name].lstrip('+') + + return Options(assemble_option_dict(options.items(), documenter.option_spec)) + + +def parse_generated_content(state: RSTState, content: StringList, documenter: Documenter + ) -> List[Node]: + """Parse an item of content generated by Documenter.""" + with switch_source_input(state, content): + if documenter.titles_allowed: + node: Element = nodes.section() + # necessary so that the child nodes get the right source/line set + node.document = state.document + nested_parse_with_titles(state, content, node) + else: + node = nodes.paragraph() + node.document = state.document + state.nested_parse(content, 0, node) + + return node.children + + +class AutodocDirective(SphinxDirective): + """A directive class for all autodoc directives. It works as a dispatcher of Documenters. + + It invokes a Documenter upon running. After the processing, it parses and returns + the content generated by Documenter. + """ + option_spec = DummyOptionSpec() + has_content = True + required_arguments = 1 + optional_arguments = 0 + final_argument_whitespace = True + + def run(self) -> List[Node]: + reporter = self.state.document.reporter + + try: + source, lineno = reporter.get_source_and_line(self.lineno) # type: ignore + except AttributeError: + source, lineno = (None, None) + logger.debug('[autodoc] %s:%s: input:\n%s', source, lineno, self.block_text) + + # look up target Documenter + objtype = self.name[4:] # strip prefix (auto-). + doccls = self.env.app.registry.documenters[objtype] + + # process the options with the selected documenter's option_spec + try: + documenter_options = process_documenter_options(doccls, self.config, self.options) + except (KeyError, ValueError, TypeError) as exc: + # an option is either unknown or has a wrong type + logger.error('An option to %s is either unknown or has an invalid value: %s' % + (self.name, exc), location=(self.env.docname, lineno)) + return [] + + # generate the output + params = DocumenterBridge(self.env, reporter, documenter_options, lineno, self.state) + documenter = doccls(params, self.arguments[0]) + documenter.generate(more_content=self.content) + if not params.result: + return [] + + logger.debug('[autodoc] output:\n%s', '\n'.join(params.result)) + + # record all filenames as dependencies -- this will at least + # partially make automatic invalidation possible + for fn in params.record_dependencies: + self.state.document.settings.record_dependencies.add(fn) + + result = parse_generated_content(self.state, params.result, documenter) + return result diff --git a/sphinx/ext/autodoc/importer.py b/sphinx/ext/autodoc/importer.py new file mode 100644 index 0000000..977cfbb --- /dev/null +++ b/sphinx/ext/autodoc/importer.py @@ -0,0 +1,301 @@ +"""Importer utilities for autodoc""" + +import importlib +import traceback +import warnings +from typing import TYPE_CHECKING, Any, Callable, Dict, List, NamedTuple, Optional + +from sphinx.ext.autodoc.mock import ismock, undecorate +from sphinx.pycode import ModuleAnalyzer, PycodeError +from sphinx.util import logging +from sphinx.util.inspect import (getannotations, getmro, getslots, isclass, isenumclass, + safe_getattr) + +if TYPE_CHECKING: + from sphinx.ext.autodoc import ObjectMember + +logger = logging.getLogger(__name__) + + +def mangle(subject: Any, name: str) -> str: + """Mangle the given name.""" + try: + if isclass(subject) and name.startswith('__') and not name.endswith('__'): + return "_%s%s" % (subject.__name__, name) + except AttributeError: + pass + + return name + + +def unmangle(subject: Any, name: str) -> Optional[str]: + """Unmangle the given name.""" + try: + if isclass(subject) and not name.endswith('__'): + prefix = "_%s__" % subject.__name__ + if name.startswith(prefix): + return name.replace(prefix, "__", 1) + else: + for cls in subject.__mro__: + prefix = "_%s__" % cls.__name__ + if name.startswith(prefix): + # mangled attribute defined in parent class + return None + except AttributeError: + pass + + return name + + +def import_module(modname: str, warningiserror: bool = False) -> Any: + """ + Call importlib.import_module(modname), convert exceptions to ImportError + """ + try: + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=ImportWarning) + with logging.skip_warningiserror(not warningiserror): + return importlib.import_module(modname) + except BaseException as exc: + # Importing modules may cause any side effects, including + # SystemExit, so we need to catch all errors. + raise ImportError(exc, traceback.format_exc()) from exc + + +def import_object(modname: str, objpath: List[str], objtype: str = '', + attrgetter: Callable[[Any, str], Any] = safe_getattr, + warningiserror: bool = False) -> Any: + if objpath: + logger.debug('[autodoc] from %s import %s', modname, '.'.join(objpath)) + else: + logger.debug('[autodoc] import %s', modname) + + try: + module = None + exc_on_importing = None + objpath = list(objpath) + while module is None: + try: + module = import_module(modname, warningiserror=warningiserror) + logger.debug('[autodoc] import %s => %r', modname, module) + except ImportError as exc: + logger.debug('[autodoc] import %s => failed', modname) + exc_on_importing = exc + if '.' in modname: + # retry with parent module + modname, name = modname.rsplit('.', 1) + objpath.insert(0, name) + else: + raise + + obj = module + parent = None + object_name = None + for attrname in objpath: + parent = obj + logger.debug('[autodoc] getattr(_, %r)', attrname) + mangled_name = mangle(obj, attrname) + obj = attrgetter(obj, mangled_name) + + try: + logger.debug('[autodoc] => %r', obj) + except TypeError: + # fallback of failure on logging for broken object + # refs: https://github.com/sphinx-doc/sphinx/issues/9095 + logger.debug('[autodoc] => %r', (obj,)) + + object_name = attrname + return [module, parent, object_name, obj] + except (AttributeError, ImportError) as exc: + if isinstance(exc, AttributeError) and exc_on_importing: + # restore ImportError + exc = exc_on_importing + + if objpath: + errmsg = ('autodoc: failed to import %s %r from module %r' % + (objtype, '.'.join(objpath), modname)) + else: + errmsg = 'autodoc: failed to import %s %r' % (objtype, modname) + + if isinstance(exc, ImportError): + # import_module() raises ImportError having real exception obj and + # traceback + real_exc, traceback_msg = exc.args + if isinstance(real_exc, SystemExit): + errmsg += ('; the module executes module level statement ' + 'and it might call sys.exit().') + elif isinstance(real_exc, ImportError) and real_exc.args: + errmsg += '; the following exception was raised:\n%s' % real_exc.args[0] + else: + errmsg += '; the following exception was raised:\n%s' % traceback_msg + else: + errmsg += '; the following exception was raised:\n%s' % traceback.format_exc() + + logger.debug(errmsg) + raise ImportError(errmsg) from exc + + +class Attribute(NamedTuple): + name: str + directly_defined: bool + value: Any + + +def get_object_members(subject: Any, objpath: List[str], attrgetter: Callable, + analyzer: ModuleAnalyzer = None) -> Dict[str, Attribute]: + """Get members and attributes of target object.""" + from sphinx.ext.autodoc import INSTANCEATTR + + # the members directly defined in the class + obj_dict = attrgetter(subject, '__dict__', {}) + + members: Dict[str, Attribute] = {} + + # enum members + if isenumclass(subject): + for name, value in subject.__members__.items(): + if name not in members: + members[name] = Attribute(name, True, value) + + superclass = subject.__mro__[1] + for name in obj_dict: + if name not in superclass.__dict__: + value = safe_getattr(subject, name) + members[name] = Attribute(name, True, value) + + # members in __slots__ + try: + __slots__ = getslots(subject) + if __slots__: + from sphinx.ext.autodoc import SLOTSATTR + + for name in __slots__: + members[name] = Attribute(name, True, SLOTSATTR) + except (TypeError, ValueError): + pass + + # other members + for name in dir(subject): + try: + value = attrgetter(subject, name) + directly_defined = name in obj_dict + name = unmangle(subject, name) + if name and name not in members: + members[name] = Attribute(name, directly_defined, value) + except AttributeError: + continue + + # annotation only member (ex. attr: int) + for i, cls in enumerate(getmro(subject)): + for name in getannotations(cls): + name = unmangle(cls, name) + if name and name not in members: + members[name] = Attribute(name, i == 0, INSTANCEATTR) + + if analyzer: + # append instance attributes (cf. self.attr1) if analyzer knows + namespace = '.'.join(objpath) + for (ns, name) in analyzer.find_attr_docs(): + if namespace == ns and name not in members: + members[name] = Attribute(name, True, INSTANCEATTR) + + return members + + +def get_class_members(subject: Any, objpath: List[str], attrgetter: Callable, + inherit_docstrings: bool = True) -> Dict[str, "ObjectMember"]: + """Get members and attributes of target class.""" + from sphinx.ext.autodoc import INSTANCEATTR, ObjectMember + + # the members directly defined in the class + obj_dict = attrgetter(subject, '__dict__', {}) + + members: Dict[str, ObjectMember] = {} + + # enum members + if isenumclass(subject): + for name, value in subject.__members__.items(): + if name not in members: + members[name] = ObjectMember(name, value, class_=subject) + + superclass = subject.__mro__[1] + for name in obj_dict: + if name not in superclass.__dict__: + value = safe_getattr(subject, name) + members[name] = ObjectMember(name, value, class_=subject) + + # members in __slots__ + try: + __slots__ = getslots(subject) + if __slots__: + from sphinx.ext.autodoc import SLOTSATTR + + for name, docstring in __slots__.items(): + members[name] = ObjectMember(name, SLOTSATTR, class_=subject, + docstring=docstring) + except (TypeError, ValueError): + pass + + # other members + for name in dir(subject): + try: + value = attrgetter(subject, name) + if ismock(value): + value = undecorate(value) + + unmangled = unmangle(subject, name) + if unmangled and unmangled not in members: + if name in obj_dict: + members[unmangled] = ObjectMember(unmangled, value, class_=subject) + else: + members[unmangled] = ObjectMember(unmangled, value) + except AttributeError: + continue + + try: + for cls in getmro(subject): + try: + modname = safe_getattr(cls, '__module__') + qualname = safe_getattr(cls, '__qualname__') + analyzer = ModuleAnalyzer.for_module(modname) + analyzer.analyze() + except AttributeError: + qualname = None + analyzer = None + except PycodeError: + analyzer = None + + # annotation only member (ex. attr: int) + for name in getannotations(cls): + name = unmangle(cls, name) + if name and name not in members: + if analyzer and (qualname, name) in analyzer.attr_docs: + docstring = '\n'.join(analyzer.attr_docs[qualname, name]) + else: + docstring = None + + members[name] = ObjectMember(name, INSTANCEATTR, class_=cls, + docstring=docstring) + + # append or complete instance attributes (cf. self.attr1) if analyzer knows + if analyzer: + for (ns, name), docstring in analyzer.attr_docs.items(): + if ns == qualname and name not in members: + # otherwise unknown instance attribute + members[name] = ObjectMember(name, INSTANCEATTR, class_=cls, + docstring='\n'.join(docstring)) + elif (ns == qualname and docstring and + isinstance(members[name], ObjectMember) and + not members[name].docstring): + if cls != subject and not inherit_docstrings: + # If we are in the MRO of the class and not the class itself, + # and we do not want to inherit docstrings, then skip setting + # the docstring below + continue + # attribute is already known, because dir(subject) enumerates it. + # But it has no docstring yet + members[name].docstring = '\n'.join(docstring) + except AttributeError: + pass + + return members diff --git a/sphinx/ext/autodoc/mock.py b/sphinx/ext/autodoc/mock.py new file mode 100644 index 0000000..15c11c3 --- /dev/null +++ b/sphinx/ext/autodoc/mock.py @@ -0,0 +1,193 @@ +"""mock for autodoc""" + +import contextlib +import os +import sys +from importlib.abc import Loader, MetaPathFinder +from importlib.machinery import ModuleSpec +from types import MethodType, ModuleType +from typing import Any, Generator, Iterator, List, Optional, Sequence, Tuple, Union + +from sphinx.util import logging +from sphinx.util.inspect import isboundmethod, safe_getattr + +logger = logging.getLogger(__name__) + + +class _MockObject: + """Used by autodoc_mock_imports.""" + + __display_name__ = '_MockObject' + __name__ = '' + __sphinx_mock__ = True + __sphinx_decorator_args__: Tuple[Any, ...] = () + + def __new__(cls, *args: Any, **kwargs: Any) -> Any: + if len(args) == 3 and isinstance(args[1], tuple): + superclass = args[1][-1].__class__ + if superclass is cls: + # subclassing MockObject + return _make_subclass(args[0], superclass.__display_name__, + superclass=superclass, attributes=args[2]) + + return super().__new__(cls) + + def __init__(self, *args: Any, **kwargs: Any) -> None: + self.__qualname__ = self.__name__ + + def __len__(self) -> int: + return 0 + + def __contains__(self, key: str) -> bool: + return False + + def __iter__(self) -> Iterator: + return iter([]) + + def __mro_entries__(self, bases: Tuple) -> Tuple: + return (self.__class__,) + + def __getitem__(self, key: Any) -> "_MockObject": + return _make_subclass(str(key), self.__display_name__, self.__class__)() + + def __getattr__(self, key: str) -> "_MockObject": + return _make_subclass(key, self.__display_name__, self.__class__)() + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + call = self.__class__() + call.__sphinx_decorator_args__ = args + return call + + def __repr__(self) -> str: + return self.__display_name__ + + +def _make_subclass(name: str, module: str, superclass: Any = _MockObject, + attributes: Any = None, decorator_args: Tuple = ()) -> Any: + attrs = {'__module__': module, + '__display_name__': module + '.' + name, + '__name__': name, + '__sphinx_decorator_args__': decorator_args} + attrs.update(attributes or {}) + + return type(name, (superclass,), attrs) + + +class _MockModule(ModuleType): + """Used by autodoc_mock_imports.""" + __file__ = os.devnull + __sphinx_mock__ = True + + def __init__(self, name: str) -> None: + super().__init__(name) + self.__all__: List[str] = [] + self.__path__: List[str] = [] + + def __getattr__(self, name: str) -> _MockObject: + return _make_subclass(name, self.__name__)() + + def __repr__(self) -> str: + return self.__name__ + + +class MockLoader(Loader): + """A loader for mocking.""" + def __init__(self, finder: "MockFinder") -> None: + super().__init__() + self.finder = finder + + def create_module(self, spec: ModuleSpec) -> ModuleType: + logger.debug('[autodoc] adding a mock module as %s!', spec.name) + self.finder.mocked_modules.append(spec.name) + return _MockModule(spec.name) + + def exec_module(self, module: ModuleType) -> None: + pass # nothing to do + + +class MockFinder(MetaPathFinder): + """A finder for mocking.""" + + def __init__(self, modnames: List[str]) -> None: + super().__init__() + self.modnames = modnames + self.loader = MockLoader(self) + self.mocked_modules: List[str] = [] + + def find_spec(self, fullname: str, path: Optional[Sequence[Union[bytes, str]]], + target: ModuleType = None) -> Optional[ModuleSpec]: + for modname in self.modnames: + # check if fullname is (or is a descendant of) one of our targets + if modname == fullname or fullname.startswith(modname + '.'): + return ModuleSpec(fullname, self.loader) + + return None + + def invalidate_caches(self) -> None: + """Invalidate mocked modules on sys.modules.""" + for modname in self.mocked_modules: + sys.modules.pop(modname, None) + + +@contextlib.contextmanager +def mock(modnames: List[str]) -> Generator[None, None, None]: + """Insert mock modules during context:: + + with mock(['target.module.name']): + # mock modules are enabled here + ... + """ + try: + finder = MockFinder(modnames) + sys.meta_path.insert(0, finder) + yield + finally: + sys.meta_path.remove(finder) + finder.invalidate_caches() + + +def ismockmodule(subject: Any) -> bool: + """Check if the object is a mocked module.""" + return isinstance(subject, _MockModule) + + +def ismock(subject: Any) -> bool: + """Check if the object is mocked.""" + # check the object has '__sphinx_mock__' attribute + try: + if safe_getattr(subject, '__sphinx_mock__', None) is None: + return False + except AttributeError: + return False + + # check the object is mocked module + if isinstance(subject, _MockModule): + return True + + # check the object is bound method + if isinstance(subject, MethodType) and isboundmethod(subject): + tmp_subject = subject.__func__ + else: + tmp_subject = subject + + try: + # check the object is mocked object + __mro__ = safe_getattr(type(tmp_subject), '__mro__', []) + if len(__mro__) > 2 and __mro__[-2] is _MockObject: + # A mocked object has a MRO that ends with (..., _MockObject, object). + return True + except AttributeError: + pass + + return False + + +def undecorate(subject: _MockObject) -> Any: + """Unwrap mock if *subject* is decorated by mocked object. + + If not decorated, returns given *subject* itself. + """ + if ismock(subject) and subject.__sphinx_decorator_args__: + return subject.__sphinx_decorator_args__[0] + else: + return subject diff --git a/sphinx/ext/autodoc/preserve_defaults.py b/sphinx/ext/autodoc/preserve_defaults.py new file mode 100644 index 0000000..5ae3f35 --- /dev/null +++ b/sphinx/ext/autodoc/preserve_defaults.py @@ -0,0 +1,127 @@ +"""Preserve function defaults. + +Preserve the default argument values of function signatures in source code +and keep them not evaluated for readability. +""" + +import ast +import inspect +import sys +from inspect import Parameter +from typing import Any, Dict, List, Optional + +import sphinx +from sphinx.application import Sphinx +from sphinx.locale import __ +from sphinx.pycode.ast import parse as ast_parse +from sphinx.pycode.ast import unparse as ast_unparse +from sphinx.util import logging + +logger = logging.getLogger(__name__) + + +class DefaultValue: + def __init__(self, name: str) -> None: + self.name = name + + def __repr__(self) -> str: + return self.name + + +def get_function_def(obj: Any) -> Optional[ast.FunctionDef]: + """Get FunctionDef object from living object. + This tries to parse original code for living object and returns + AST node for given *obj*. + """ + try: + source = inspect.getsource(obj) + if source.startswith((' ', r'\t')): + # subject is placed inside class or block. To read its docstring, + # this adds if-block before the declaration. + module = ast_parse('if True:\n' + source) + return module.body[0].body[0] # type: ignore + else: + module = ast_parse(source) + return module.body[0] # type: ignore + except (OSError, TypeError): # failed to load source code + return None + + +def get_default_value(lines: List[str], position: ast.AST) -> Optional[str]: + try: + if sys.version_info < (3, 8): # only for py38+ + return None + elif position.lineno == position.end_lineno: + line = lines[position.lineno - 1] + return line[position.col_offset:position.end_col_offset] + else: + # multiline value is not supported now + return None + except (AttributeError, IndexError): + return None + + +def update_defvalue(app: Sphinx, obj: Any, bound_method: bool) -> None: + """Update defvalue info of *obj* using type_comments.""" + if not app.config.autodoc_preserve_defaults: + return + + try: + lines = inspect.getsource(obj).splitlines() + if lines[0].startswith((' ', r'\t')): + lines.insert(0, '') # insert a dummy line to follow what get_function_def() does. + except (OSError, TypeError): + lines = [] + + try: + function = get_function_def(obj) + if function.args.defaults or function.args.kw_defaults: + sig = inspect.signature(obj) + defaults = list(function.args.defaults) + kw_defaults = list(function.args.kw_defaults) + parameters = list(sig.parameters.values()) + for i, param in enumerate(parameters): + if param.default is param.empty: + if param.kind == param.KEYWORD_ONLY: + # Consume kw_defaults for kwonly args + kw_defaults.pop(0) + else: + if param.kind in (param.POSITIONAL_ONLY, param.POSITIONAL_OR_KEYWORD): + default = defaults.pop(0) + value = get_default_value(lines, default) + if value is None: + value = ast_unparse(default) # type: ignore + parameters[i] = param.replace(default=DefaultValue(value)) + else: + default = kw_defaults.pop(0) + value = get_default_value(lines, default) + if value is None: + value = ast_unparse(default) # type: ignore + parameters[i] = param.replace(default=DefaultValue(value)) + + if bound_method and inspect.ismethod(obj): + # classmethods + cls = inspect.Parameter('cls', Parameter.POSITIONAL_OR_KEYWORD) + parameters.insert(0, cls) + + sig = sig.replace(parameters=parameters) + if bound_method and inspect.ismethod(obj): + # classmethods can't be assigned __signature__ attribute. + obj.__dict__['__signature__'] = sig + else: + obj.__signature__ = sig + except (AttributeError, TypeError): + # failed to update signature (ex. built-in or extension types) + pass + except NotImplementedError as exc: # failed to ast.unparse() + logger.warning(__("Failed to parse a default argument value for %r: %s"), obj, exc) + + +def setup(app: Sphinx) -> Dict[str, Any]: + app.add_config_value('autodoc_preserve_defaults', False, True) + app.connect('autodoc-before-process-signature', update_defvalue) + + return { + 'version': sphinx.__display_version__, + 'parallel_read_safe': True + } diff --git a/sphinx/ext/autodoc/type_comment.py b/sphinx/ext/autodoc/type_comment.py new file mode 100644 index 0000000..9881ae7 --- /dev/null +++ b/sphinx/ext/autodoc/type_comment.py @@ -0,0 +1,131 @@ +"""Update annotations info of living objects using type_comments.""" + +from inspect import Parameter, Signature, getsource +from typing import Any, Dict, List, cast + +import sphinx +from sphinx.application import Sphinx +from sphinx.locale import __ +from sphinx.pycode.ast import ast +from sphinx.pycode.ast import parse as ast_parse +from sphinx.pycode.ast import unparse as ast_unparse +from sphinx.util import inspect, logging + +logger = logging.getLogger(__name__) + + +def not_suppressed(argtypes: List[ast.AST] = []) -> bool: + """Check given *argtypes* is suppressed type_comment or not.""" + if len(argtypes) == 0: # no argtypees + return False + elif len(argtypes) == 1 and ast_unparse(argtypes[0]) == "...": # suppressed + # Note: To support multiple versions of python, this uses ``ast_unparse()`` for + # comparison with Ellipsis. Since 3.8, ast.Constant has been used to represent + # Ellipsis node instead of ast.Ellipsis. + return False + else: # not suppressed + return True + + +def signature_from_ast(node: ast.FunctionDef, bound_method: bool, + type_comment: ast.FunctionDef) -> Signature: + """Return a Signature object for the given *node*. + + :param bound_method: Specify *node* is a bound method or not + """ + params = [] + if hasattr(node.args, "posonlyargs"): # for py38+ + for arg in node.args.posonlyargs: # type: ignore + param = Parameter(arg.arg, Parameter.POSITIONAL_ONLY, annotation=arg.type_comment) + params.append(param) + + for arg in node.args.args: + param = Parameter(arg.arg, Parameter.POSITIONAL_OR_KEYWORD, + annotation=arg.type_comment or Parameter.empty) + params.append(param) + + if node.args.vararg: + param = Parameter(node.args.vararg.arg, Parameter.VAR_POSITIONAL, + annotation=node.args.vararg.type_comment or Parameter.empty) + params.append(param) + + for arg in node.args.kwonlyargs: + param = Parameter(arg.arg, Parameter.KEYWORD_ONLY, + annotation=arg.type_comment or Parameter.empty) + params.append(param) + + if node.args.kwarg: + param = Parameter(node.args.kwarg.arg, Parameter.VAR_KEYWORD, + annotation=node.args.kwarg.type_comment or Parameter.empty) + params.append(param) + + # Remove first parameter when *obj* is bound_method + if bound_method and params: + params.pop(0) + + # merge type_comment into signature + if not_suppressed(type_comment.argtypes): # type: ignore + for i, param in enumerate(params): + params[i] = param.replace(annotation=type_comment.argtypes[i]) # type: ignore + + if node.returns: + return Signature(params, return_annotation=node.returns) + elif type_comment.returns: + return Signature(params, return_annotation=ast_unparse(type_comment.returns)) + else: + return Signature(params) + + +def get_type_comment(obj: Any, bound_method: bool = False) -> Signature: + """Get type_comment'ed FunctionDef object from living object. + + This tries to parse original code for living object and returns + Signature for given *obj*. It requires py38+ or typed_ast module. + """ + try: + source = getsource(obj) + if source.startswith((' ', r'\t')): + # subject is placed inside class or block. To read its docstring, + # this adds if-block before the declaration. + module = ast_parse('if True:\n' + source) + subject = cast(ast.FunctionDef, module.body[0].body[0]) # type: ignore + else: + module = ast_parse(source) + subject = cast(ast.FunctionDef, module.body[0]) # type: ignore + + if getattr(subject, "type_comment", None): + function = ast_parse(subject.type_comment, mode='func_type') + return signature_from_ast(subject, bound_method, function) # type: ignore + else: + return None + except (OSError, TypeError): # failed to load source code + return None + except SyntaxError: # failed to parse type_comments + return None + + +def update_annotations_using_type_comments(app: Sphinx, obj: Any, bound_method: bool) -> None: + """Update annotations info of *obj* using type_comments.""" + try: + type_sig = get_type_comment(obj, bound_method) + if type_sig: + sig = inspect.signature(obj, bound_method) + for param in sig.parameters.values(): + if param.name not in obj.__annotations__: + annotation = type_sig.parameters[param.name].annotation + if annotation is not Parameter.empty: + obj.__annotations__[param.name] = ast_unparse(annotation) + + if 'return' not in obj.__annotations__: + obj.__annotations__['return'] = type_sig.return_annotation + except KeyError as exc: + logger.warning(__("Failed to update signature for %r: parameter not found: %s"), + obj, exc) + except NotImplementedError as exc: # failed to ast.unparse() + logger.warning(__("Failed to parse type_comment for %r: %s"), obj, exc) + + +def setup(app: Sphinx) -> Dict[str, Any]: + app.connect('autodoc-before-process-signature', update_annotations_using_type_comments) + + return {'version': sphinx.__display_version__, 'parallel_read_safe': True} diff --git a/sphinx/ext/autodoc/typehints.py b/sphinx/ext/autodoc/typehints.py new file mode 100644 index 0000000..2acacfe --- /dev/null +++ b/sphinx/ext/autodoc/typehints.py @@ -0,0 +1,213 @@ +"""Generating content for autodoc using typehints""" + +import re +from collections import OrderedDict +from typing import Any, Dict, Iterable, Set, cast + +from docutils import nodes +from docutils.nodes import Element + +import sphinx +from sphinx import addnodes +from sphinx.application import Sphinx +from sphinx.util import inspect, typing + + +def record_typehints(app: Sphinx, objtype: str, name: str, obj: Any, + options: Dict, args: str, retann: str) -> None: + """Record type hints to env object.""" + if app.config.autodoc_typehints_format == 'short': + mode = 'smart' + else: + mode = 'fully-qualified' + + try: + if callable(obj): + annotations = app.env.temp_data.setdefault('annotations', {}) + annotation = annotations.setdefault(name, OrderedDict()) + sig = inspect.signature(obj, type_aliases=app.config.autodoc_type_aliases) + for param in sig.parameters.values(): + if param.annotation is not param.empty: + annotation[param.name] = typing.stringify(param.annotation, mode) + if sig.return_annotation is not sig.empty: + annotation['return'] = typing.stringify(sig.return_annotation, mode) + except (TypeError, ValueError): + pass + + +def merge_typehints(app: Sphinx, domain: str, objtype: str, contentnode: Element) -> None: + if domain != 'py': + return + if app.config.autodoc_typehints not in ('both', 'description'): + return + + try: + signature = cast(addnodes.desc_signature, contentnode.parent[0]) + if signature['module']: + fullname = '.'.join([signature['module'], signature['fullname']]) + else: + fullname = signature['fullname'] + except KeyError: + # signature node does not have valid context info for the target object + return + + annotations = app.env.temp_data.get('annotations', {}) + if annotations.get(fullname, {}): + field_lists = [n for n in contentnode if isinstance(n, nodes.field_list)] + if field_lists == []: + field_list = insert_field_list(contentnode) + field_lists.append(field_list) + + for field_list in field_lists: + if app.config.autodoc_typehints_description_target == "all": + if objtype == 'class': + modify_field_list(field_list, annotations[fullname], suppress_rtype=True) + else: + modify_field_list(field_list, annotations[fullname]) + elif app.config.autodoc_typehints_description_target == "documented_params": + augment_descriptions_with_types( + field_list, annotations[fullname], force_rtype=True + ) + else: + augment_descriptions_with_types( + field_list, annotations[fullname], force_rtype=False + ) + + +def insert_field_list(node: Element) -> nodes.field_list: + field_list = nodes.field_list() + desc = [n for n in node if isinstance(n, addnodes.desc)] + if desc: + # insert just before sub object descriptions (ex. methods, nested classes, etc.) + index = node.index(desc[0]) + node.insert(index - 1, [field_list]) + else: + node += field_list + + return field_list + + +def modify_field_list(node: nodes.field_list, annotations: Dict[str, str], + suppress_rtype: bool = False) -> None: + arguments: Dict[str, Dict[str, bool]] = {} + fields = cast(Iterable[nodes.field], node) + for field in fields: + field_name = field[0].astext() + parts = re.split(' +', field_name) + if parts[0] == 'param': + if len(parts) == 2: + # :param xxx: + arg = arguments.setdefault(parts[1], {}) + arg['param'] = True + elif len(parts) > 2: + # :param xxx yyy: + name = ' '.join(parts[2:]) + arg = arguments.setdefault(name, {}) + arg['param'] = True + arg['type'] = True + elif parts[0] == 'type': + name = ' '.join(parts[1:]) + arg = arguments.setdefault(name, {}) + arg['type'] = True + elif parts[0] == 'rtype': + arguments['return'] = {'type': True} + + for name, annotation in annotations.items(): + if name == 'return': + continue + + if '*' + name in arguments: + name = '*' + name + arguments.get(name) + elif '**' + name in arguments: + name = '**' + name + arguments.get(name) + else: + arg = arguments.get(name, {}) + + if not arg.get('type'): + field = nodes.field() + field += nodes.field_name('', 'type ' + name) + field += nodes.field_body('', nodes.paragraph('', annotation)) + node += field + if not arg.get('param'): + field = nodes.field() + field += nodes.field_name('', 'param ' + name) + field += nodes.field_body('', nodes.paragraph('', '')) + node += field + + if 'return' in annotations and 'return' not in arguments: + annotation = annotations['return'] + if annotation == 'None' and suppress_rtype: + return + + field = nodes.field() + field += nodes.field_name('', 'rtype') + field += nodes.field_body('', nodes.paragraph('', annotation)) + node += field + + +def augment_descriptions_with_types( + node: nodes.field_list, + annotations: Dict[str, str], + force_rtype: bool +) -> None: + fields = cast(Iterable[nodes.field], node) + has_description = set() # type: Set[str] + has_type = set() # type: Set[str] + for field in fields: + field_name = field[0].astext() + parts = re.split(' +', field_name) + if parts[0] == 'param': + if len(parts) == 2: + # :param xxx: + has_description.add(parts[1]) + elif len(parts) > 2: + # :param xxx yyy: + name = ' '.join(parts[2:]) + has_description.add(name) + has_type.add(name) + elif parts[0] == 'type': + name = ' '.join(parts[1:]) + has_type.add(name) + elif parts[0] in ('return', 'returns'): + has_description.add('return') + elif parts[0] == 'rtype': + has_type.add('return') + + # Add 'type' for parameters with a description but no declared type. + for name, annotation in annotations.items(): + if name in ('return', 'returns'): + continue + + if '*' + name in has_description: + name = '*' + name + elif '**' + name in has_description: + name = '**' + name + + if name in has_description and name not in has_type: + field = nodes.field() + field += nodes.field_name('', 'type ' + name) + field += nodes.field_body('', nodes.paragraph('', annotation)) + node += field + + # Add 'rtype' if 'return' is present and 'rtype' isn't. + if 'return' in annotations: + rtype = annotations['return'] + if 'return' not in has_type and ('return' in has_description or + (force_rtype and rtype != "None")): + field = nodes.field() + field += nodes.field_name('', 'rtype') + field += nodes.field_body('', nodes.paragraph('', rtype)) + node += field + + +def setup(app: Sphinx) -> Dict[str, Any]: + app.connect('autodoc-process-signature', record_typehints) + app.connect('object-description-transform', merge_typehints) + + return { + 'version': sphinx.__display_version__, + 'parallel_read_safe': True, + 'parallel_write_safe': True, + } |