diff options
Diffstat (limited to 'sphinx/ext/autosummary/generate.py')
| -rw-r--r-- | sphinx/ext/autosummary/generate.py | 754 |
1 files changed, 754 insertions, 0 deletions
diff --git a/sphinx/ext/autosummary/generate.py b/sphinx/ext/autosummary/generate.py new file mode 100644 index 0000000..06814f9 --- /dev/null +++ b/sphinx/ext/autosummary/generate.py @@ -0,0 +1,754 @@ +"""Generates reST source files for autosummary. + +Usable as a library or script to generate automatic RST source files for +items referred to in autosummary:: directives. + +Each generated RST file contains a single auto*:: directive which +extracts the docstring of the referred item. + +Example Makefile rule:: + + generate: + sphinx-autogen -o source/generated source/*.rst +""" + +from __future__ import annotations + +import argparse +import importlib +import inspect +import locale +import os +import pkgutil +import pydoc +import re +import sys +from os import path +from typing import TYPE_CHECKING, Any, NamedTuple + +from jinja2 import TemplateNotFound +from jinja2.sandbox import SandboxedEnvironment + +import sphinx.locale +from sphinx import __display_version__, package_dir +from sphinx.builders import Builder +from sphinx.config import Config +from sphinx.ext.autodoc.importer import import_module +from sphinx.ext.autosummary import ( + ImportExceptionGroup, + get_documenter, + import_by_name, + import_ivar_by_name, +) +from sphinx.locale import __ +from sphinx.pycode import ModuleAnalyzer, PycodeError +from sphinx.registry import SphinxComponentRegistry +from sphinx.util import logging, rst +from sphinx.util.inspect import getall, safe_getattr +from sphinx.util.osutil import ensuredir +from sphinx.util.template import SphinxTemplateLoader + +if TYPE_CHECKING: + from collections.abc import Sequence, Set + from gettext import NullTranslations + + from sphinx.application import Sphinx + from sphinx.ext.autodoc import Documenter + +logger = logging.getLogger(__name__) + + +class DummyApplication: + """Dummy Application class for sphinx-autogen command.""" + + def __init__(self, translator: NullTranslations) -> None: + self.config = Config() + self.registry = SphinxComponentRegistry() + self.messagelog: list[str] = [] + self.srcdir = "/" + self.translator = translator + self.verbosity = 0 + self._warncount = 0 + self.warningiserror = False + + self.config.add('autosummary_context', {}, True, None) + self.config.add('autosummary_filename_map', {}, True, None) + self.config.add('autosummary_ignore_module_all', True, 'env', bool) + self.config.init_values() + + def emit_firstresult(self, *args: Any) -> None: + pass + + +class AutosummaryEntry(NamedTuple): + name: str + path: str | None + template: str + recursive: bool + + +def setup_documenters(app: Any) -> None: + from sphinx.ext.autodoc import ( + AttributeDocumenter, + ClassDocumenter, + DataDocumenter, + DecoratorDocumenter, + ExceptionDocumenter, + FunctionDocumenter, + MethodDocumenter, + ModuleDocumenter, + PropertyDocumenter, + ) + documenters: list[type[Documenter]] = [ + ModuleDocumenter, ClassDocumenter, ExceptionDocumenter, DataDocumenter, + FunctionDocumenter, MethodDocumenter, + AttributeDocumenter, DecoratorDocumenter, PropertyDocumenter, + ] + for documenter in documenters: + app.registry.add_documenter(documenter.objtype, documenter) + + +def _underline(title: str, line: str = '=') -> str: + if '\n' in title: + msg = 'Can only underline single lines' + raise ValueError(msg) + return title + '\n' + line * len(title) + + +class AutosummaryRenderer: + """A helper class for rendering.""" + + def __init__(self, app: Sphinx) -> None: + if isinstance(app, Builder): + msg = 'Expected a Sphinx application object!' + raise ValueError(msg) + + system_templates_path = [os.path.join(package_dir, 'ext', 'autosummary', 'templates')] + loader = SphinxTemplateLoader(app.srcdir, app.config.templates_path, + system_templates_path) + + self.env = SandboxedEnvironment(loader=loader) + self.env.filters['escape'] = rst.escape + self.env.filters['e'] = rst.escape + self.env.filters['underline'] = _underline + + if app.translator: + self.env.add_extension("jinja2.ext.i18n") + self.env.install_gettext_translations(app.translator) + + def render(self, template_name: str, context: dict) -> str: + """Render a template file.""" + try: + template = self.env.get_template(template_name) + except TemplateNotFound: + try: + # objtype is given as template_name + template = self.env.get_template('autosummary/%s.rst' % template_name) + except TemplateNotFound: + # fallback to base.rst + template = self.env.get_template('autosummary/base.rst') + + return template.render(context) + + +def _split_full_qualified_name(name: str) -> tuple[str | None, str]: + """Split full qualified name to a pair of modname and qualname. + + A qualname is an abbreviation for "Qualified name" introduced at PEP-3155 + (https://peps.python.org/pep-3155/). It is a dotted path name + from the module top-level. + + A "full" qualified name means a string containing both module name and + qualified name. + + .. note:: This function actually imports the module to check its existence. + Therefore you need to mock 3rd party modules if needed before + calling this function. + """ + parts = name.split('.') + for i, _part in enumerate(parts, 1): + try: + modname = ".".join(parts[:i]) + importlib.import_module(modname) + except ImportError: + if parts[:i - 1]: + return ".".join(parts[:i - 1]), ".".join(parts[i - 1:]) + else: + return None, ".".join(parts) + except IndexError: + pass + + return name, "" + + +# -- Generating output --------------------------------------------------------- + + +class ModuleScanner: + def __init__(self, app: Any, obj: Any) -> None: + self.app = app + self.object = obj + + def get_object_type(self, name: str, value: Any) -> str: + return get_documenter(self.app, value, self.object).objtype + + def is_skipped(self, name: str, value: Any, objtype: str) -> bool: + try: + return self.app.emit_firstresult('autodoc-skip-member', objtype, + name, value, False, {}) + except Exception as exc: + logger.warning(__('autosummary: failed to determine %r to be documented, ' + 'the following exception was raised:\n%s'), + name, exc, type='autosummary') + return False + + def scan(self, imported_members: bool) -> list[str]: + members = [] + try: + analyzer = ModuleAnalyzer.for_module(self.object.__name__) + attr_docs = analyzer.find_attr_docs() + except PycodeError: + attr_docs = {} + + for name in members_of(self.object, self.app.config): + try: + value = safe_getattr(self.object, name) + except AttributeError: + value = None + + objtype = self.get_object_type(name, value) + if self.is_skipped(name, value, objtype): + continue + + try: + if ('', name) in attr_docs: + imported = False + elif inspect.ismodule(value): # NoQA: SIM114 + imported = True + elif safe_getattr(value, '__module__') != self.object.__name__: + imported = True + else: + imported = False + except AttributeError: + imported = False + + respect_module_all = not self.app.config.autosummary_ignore_module_all + if ( + # list all members up + imported_members + # list not-imported members + or imported is False + # list members that have __all__ set + or (respect_module_all and '__all__' in dir(self.object)) + ): + members.append(name) + + return members + + +def members_of(obj: Any, conf: Config) -> Sequence[str]: + """Get the members of ``obj``, possibly ignoring the ``__all__`` module attribute + + Follows the ``conf.autosummary_ignore_module_all`` setting.""" + + if conf.autosummary_ignore_module_all: + return dir(obj) + else: + return getall(obj) or dir(obj) + + +def generate_autosummary_content(name: str, obj: Any, parent: Any, + template: AutosummaryRenderer, template_name: str, + imported_members: bool, app: Any, + recursive: bool, context: dict, + modname: str | None = None, + qualname: str | None = None) -> str: + doc = get_documenter(app, obj, parent) + + ns: dict[str, Any] = {} + ns.update(context) + + if doc.objtype == 'module': + scanner = ModuleScanner(app, obj) + ns['members'] = scanner.scan(imported_members) + + respect_module_all = not app.config.autosummary_ignore_module_all + imported_members = imported_members or ('__all__' in dir(obj) and respect_module_all) + + ns['functions'], ns['all_functions'] = \ + _get_members(doc, app, obj, {'function'}, imported=imported_members) + ns['classes'], ns['all_classes'] = \ + _get_members(doc, app, obj, {'class'}, imported=imported_members) + ns['exceptions'], ns['all_exceptions'] = \ + _get_members(doc, app, obj, {'exception'}, imported=imported_members) + ns['attributes'], ns['all_attributes'] = \ + _get_module_attrs(name, ns['members']) + ispackage = hasattr(obj, '__path__') + if ispackage and recursive: + # Use members that are not modules as skip list, because it would then mean + # that module was overwritten in the package namespace + skip = ( + ns["all_functions"] + + ns["all_classes"] + + ns["all_exceptions"] + + ns["all_attributes"] + ) + + # If respect_module_all and module has a __all__ attribute, first get + # modules that were explicitly imported. Next, find the rest with the + # get_modules method, but only put in "public" modules that are in the + # __all__ list + # + # Otherwise, use get_modules method normally + if respect_module_all and '__all__' in dir(obj): + imported_modules, all_imported_modules = \ + _get_members(doc, app, obj, {'module'}, imported=True) + skip += all_imported_modules + imported_modules = [name + '.' + modname for modname in imported_modules] + all_imported_modules = \ + [name + '.' + modname for modname in all_imported_modules] + public_members = getall(obj) + else: + imported_modules, all_imported_modules = [], [] + public_members = None + + modules, all_modules = _get_modules(obj, skip=skip, name=name, + public_members=public_members) + ns['modules'] = imported_modules + modules + ns["all_modules"] = all_imported_modules + all_modules + elif doc.objtype == 'class': + ns['members'] = dir(obj) + ns['inherited_members'] = \ + set(dir(obj)) - set(obj.__dict__.keys()) + ns['methods'], ns['all_methods'] = \ + _get_members(doc, app, obj, {'method'}, include_public={'__init__'}) + ns['attributes'], ns['all_attributes'] = \ + _get_members(doc, app, obj, {'attribute', 'property'}) + + if modname is None or qualname is None: + modname, qualname = _split_full_qualified_name(name) + + if doc.objtype in ('method', 'attribute', 'property'): + ns['class'] = qualname.rsplit(".", 1)[0] + + if doc.objtype in ('class',): + shortname = qualname + else: + shortname = qualname.rsplit(".", 1)[-1] + + ns['fullname'] = name + ns['module'] = modname + ns['objname'] = qualname + ns['name'] = shortname + + ns['objtype'] = doc.objtype + ns['underline'] = len(name) * '=' + + if template_name: + return template.render(template_name, ns) + else: + return template.render(doc.objtype, ns) + + +def _skip_member(app: Sphinx, obj: Any, name: str, objtype: str) -> bool: + try: + return app.emit_firstresult('autodoc-skip-member', objtype, name, + obj, False, {}) + except Exception as exc: + logger.warning(__('autosummary: failed to determine %r to be documented, ' + 'the following exception was raised:\n%s'), + name, exc, type='autosummary') + return False + + +def _get_class_members(obj: Any) -> dict[str, Any]: + members = sphinx.ext.autodoc.get_class_members(obj, None, safe_getattr) + return {name: member.object for name, member in members.items()} + + +def _get_module_members(app: Sphinx, obj: Any) -> dict[str, Any]: + members = {} + for name in members_of(obj, app.config): + try: + members[name] = safe_getattr(obj, name) + except AttributeError: + continue + return members + + +def _get_all_members(doc: type[Documenter], app: Sphinx, obj: Any) -> dict[str, Any]: + if doc.objtype == 'module': + return _get_module_members(app, obj) + elif doc.objtype == 'class': + return _get_class_members(obj) + return {} + + +def _get_members(doc: type[Documenter], app: Sphinx, obj: Any, types: set[str], *, + include_public: Set[str] = frozenset(), + imported: bool = True) -> tuple[list[str], list[str]]: + items: list[str] = [] + public: list[str] = [] + + all_members = _get_all_members(doc, app, obj) + for name, value in all_members.items(): + documenter = get_documenter(app, value, obj) + if documenter.objtype in types: + # skip imported members if expected + if imported or getattr(value, '__module__', None) == obj.__name__: + skipped = _skip_member(app, value, name, documenter.objtype) + if skipped is True: + pass + elif skipped is False: + # show the member forcedly + items.append(name) + public.append(name) + else: + items.append(name) + if name in include_public or not name.startswith('_'): + # considers member as public + public.append(name) + return public, items + + +def _get_module_attrs(name: str, members: Any) -> tuple[list[str], list[str]]: + """Find module attributes with docstrings.""" + attrs, public = [], [] + try: + analyzer = ModuleAnalyzer.for_module(name) + attr_docs = analyzer.find_attr_docs() + for namespace, attr_name in attr_docs: + if namespace == '' and attr_name in members: + attrs.append(attr_name) + if not attr_name.startswith('_'): + public.append(attr_name) + except PycodeError: + pass # give up if ModuleAnalyzer fails to parse code + return public, attrs + + +def _get_modules( + obj: Any, + *, + skip: Sequence[str], + name: str, + public_members: Sequence[str] | None = None) -> tuple[list[str], list[str]]: + items: list[str] = [] + public: list[str] = [] + for _, modname, _ispkg in pkgutil.iter_modules(obj.__path__): + + if modname in skip: + # module was overwritten in __init__.py, so not accessible + continue + fullname = name + '.' + modname + try: + module = import_module(fullname) + if module and hasattr(module, '__sphinx_mock__'): + continue + except ImportError: + pass + + items.append(fullname) + if public_members is not None: + if modname in public_members: + public.append(fullname) + else: + if not modname.startswith('_'): + public.append(fullname) + return public, items + + +def generate_autosummary_docs(sources: list[str], + output_dir: str | os.PathLike[str] | None = None, + suffix: str = '.rst', + base_path: str | os.PathLike[str] | None = None, + imported_members: bool = False, app: Any = None, + overwrite: bool = True, encoding: str = 'utf-8') -> None: + showed_sources = sorted(sources) + if len(showed_sources) > 20: + showed_sources = showed_sources[:10] + ['...'] + showed_sources[-10:] + logger.info(__('[autosummary] generating autosummary for: %s') % + ', '.join(showed_sources)) + + if output_dir: + logger.info(__('[autosummary] writing to %s') % output_dir) + + if base_path is not None: + sources = [os.path.join(base_path, filename) for filename in sources] + + template = AutosummaryRenderer(app) + + # read + items = find_autosummary_in_files(sources) + + # keep track of new files + new_files = [] + + if app: + filename_map = app.config.autosummary_filename_map + else: + filename_map = {} + + # write + for entry in sorted(set(items), key=str): + if entry.path is None: + # The corresponding autosummary:: directive did not have + # a :toctree: option + continue + + path = output_dir or os.path.abspath(entry.path) + ensuredir(path) + + try: + name, obj, parent, modname = import_by_name(entry.name) + qualname = name.replace(modname + ".", "") + except ImportExceptionGroup as exc: + try: + # try to import as an instance attribute + name, obj, parent, modname = import_ivar_by_name(entry.name) + qualname = name.replace(modname + ".", "") + except ImportError as exc2: + if exc2.__cause__: + exceptions: list[BaseException] = exc.exceptions + [exc2.__cause__] + else: + exceptions = exc.exceptions + [exc2] + + errors = list({f"* {type(e).__name__}: {e}" for e in exceptions}) + logger.warning(__('[autosummary] failed to import %s.\nPossible hints:\n%s'), + entry.name, '\n'.join(errors)) + continue + + context: dict[str, Any] = {} + if app: + context.update(app.config.autosummary_context) + + content = generate_autosummary_content(name, obj, parent, template, entry.template, + imported_members, app, entry.recursive, context, + modname, qualname) + + filename = os.path.join(path, filename_map.get(name, name) + suffix) + if os.path.isfile(filename): + with open(filename, encoding=encoding) as f: + old_content = f.read() + + if content == old_content: + continue + if overwrite: # content has changed + with open(filename, 'w', encoding=encoding) as f: + f.write(content) + new_files.append(filename) + else: + with open(filename, 'w', encoding=encoding) as f: + f.write(content) + new_files.append(filename) + + # descend recursively to new files + if new_files: + generate_autosummary_docs(new_files, output_dir=output_dir, + suffix=suffix, base_path=base_path, + imported_members=imported_members, app=app, + overwrite=overwrite) + + +# -- Finding documented entries in files --------------------------------------- + +def find_autosummary_in_files(filenames: list[str]) -> list[AutosummaryEntry]: + """Find out what items are documented in source/*.rst. + + See `find_autosummary_in_lines`. + """ + documented: list[AutosummaryEntry] = [] + for filename in filenames: + with open(filename, encoding='utf-8', errors='ignore') as f: + lines = f.read().splitlines() + documented.extend(find_autosummary_in_lines(lines, filename=filename)) + return documented + + +def find_autosummary_in_docstring( + name: str, filename: str | None = None, +) -> list[AutosummaryEntry]: + """Find out what items are documented in the given object's docstring. + + See `find_autosummary_in_lines`. + """ + try: + real_name, obj, parent, modname = import_by_name(name) + lines = pydoc.getdoc(obj).splitlines() + return find_autosummary_in_lines(lines, module=name, filename=filename) + except AttributeError: + pass + except ImportExceptionGroup as exc: + errors = '\n'.join({f"* {type(e).__name__}: {e}" for e in exc.exceptions}) + logger.warning(f'Failed to import {name}.\nPossible hints:\n{errors}') # NoQA: G004 + except SystemExit: + logger.warning("Failed to import '%s'; the module executes module level " + 'statement and it might call sys.exit().', name) + return [] + + +def find_autosummary_in_lines( + lines: list[str], module: str | None = None, filename: str | None = None, +) -> list[AutosummaryEntry]: + """Find out what items appear in autosummary:: directives in the + given lines. + + Returns a list of (name, toctree, template) where *name* is a name + of an object and *toctree* the :toctree: path of the corresponding + autosummary directive (relative to the root of the file name), and + *template* the value of the :template: option. *toctree* and + *template* ``None`` if the directive does not have the + corresponding options set. + """ + autosummary_re = re.compile(r'^(\s*)\.\.\s+autosummary::\s*') + automodule_re = re.compile( + r'^\s*\.\.\s+automodule::\s*([A-Za-z0-9_.]+)\s*$') + module_re = re.compile( + r'^\s*\.\.\s+(current)?module::\s*([a-zA-Z0-9_.]+)\s*$') + autosummary_item_re = re.compile(r'^\s+(~?[_a-zA-Z][a-zA-Z0-9_.]*)\s*.*?') + recursive_arg_re = re.compile(r'^\s+:recursive:\s*$') + toctree_arg_re = re.compile(r'^\s+:toctree:\s*(.*?)\s*$') + template_arg_re = re.compile(r'^\s+:template:\s*(.*?)\s*$') + + documented: list[AutosummaryEntry] = [] + + recursive = False + toctree: str | None = None + template = '' + current_module = module + in_autosummary = False + base_indent = "" + + for line in lines: + if in_autosummary: + m = recursive_arg_re.match(line) + if m: + recursive = True + continue + + m = toctree_arg_re.match(line) + if m: + toctree = m.group(1) + if filename: + toctree = os.path.join(os.path.dirname(filename), + toctree) + continue + + m = template_arg_re.match(line) + if m: + template = m.group(1).strip() + continue + + if line.strip().startswith(':'): + continue # skip options + + m = autosummary_item_re.match(line) + if m: + name = m.group(1).strip() + if name.startswith('~'): + name = name[1:] + if current_module and \ + not name.startswith(current_module + '.'): + name = f"{current_module}.{name}" + documented.append(AutosummaryEntry(name, toctree, template, recursive)) + continue + + if not line.strip() or line.startswith(base_indent + " "): + continue + + in_autosummary = False + + m = autosummary_re.match(line) + if m: + in_autosummary = True + base_indent = m.group(1) + recursive = False + toctree = None + template = '' + continue + + m = automodule_re.search(line) + if m: + current_module = m.group(1).strip() + # recurse into the automodule docstring + documented.extend(find_autosummary_in_docstring( + current_module, filename=filename)) + continue + + m = module_re.match(line) + if m: + current_module = m.group(2) + continue + + return documented + + +def get_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + usage='%(prog)s [OPTIONS] <SOURCE_FILE>...', + epilog=__('For more information, visit <https://www.sphinx-doc.org/>.'), + description=__(""" +Generate ReStructuredText using autosummary directives. + +sphinx-autogen is a frontend to sphinx.ext.autosummary.generate. It generates +the reStructuredText files from the autosummary directives contained in the +given input files. + +The format of the autosummary directive is documented in the +``sphinx.ext.autosummary`` Python module and can be read using:: + + pydoc sphinx.ext.autosummary +""")) + + parser.add_argument('--version', action='version', dest='show_version', + version='%%(prog)s %s' % __display_version__) + + parser.add_argument('source_file', nargs='+', + help=__('source files to generate rST files for')) + + parser.add_argument('-o', '--output-dir', action='store', + dest='output_dir', + help=__('directory to place all output in')) + parser.add_argument('-s', '--suffix', action='store', dest='suffix', + default='rst', + help=__('default suffix for files (default: ' + '%(default)s)')) + parser.add_argument('-t', '--templates', action='store', dest='templates', + default=None, + help=__('custom template directory (default: ' + '%(default)s)')) + parser.add_argument('-i', '--imported-members', action='store_true', + dest='imported_members', default=False, + help=__('document imported members (default: ' + '%(default)s)')) + parser.add_argument('-a', '--respect-module-all', action='store_true', + dest='respect_module_all', default=False, + help=__('document exactly the members in module __all__ attribute. ' + '(default: %(default)s)')) + + return parser + + +def main(argv: Sequence[str] = (), /) -> None: + locale.setlocale(locale.LC_ALL, '') + sphinx.locale.init_console() + + app = DummyApplication(sphinx.locale.get_translator()) + logging.setup(app, sys.stdout, sys.stderr) # type: ignore[arg-type] + setup_documenters(app) + args = get_parser().parse_args(argv or sys.argv[1:]) + + if args.templates: + app.config.templates_path.append(path.abspath(args.templates)) + app.config.autosummary_ignore_module_all = ( # type: ignore[attr-defined] + not args.respect_module_all + ) + + generate_autosummary_docs(args.source_file, args.output_dir, + '.' + args.suffix, + imported_members=args.imported_members, + app=app) + + +if __name__ == '__main__': + main(sys.argv[1:]) |