diff options
Diffstat (limited to 'sphinx/ext/doctest.py')
| -rw-r--r-- | sphinx/ext/doctest.py | 575 |
1 files changed, 575 insertions, 0 deletions
diff --git a/sphinx/ext/doctest.py b/sphinx/ext/doctest.py new file mode 100644 index 0000000..c55ef2f --- /dev/null +++ b/sphinx/ext/doctest.py @@ -0,0 +1,575 @@ +"""Mimic doctest in Sphinx. + +The extension automatically execute code snippets and checks their results. +""" + +from __future__ import annotations + +import doctest +import re +import sys +import time +from io import StringIO +from os import path +from typing import TYPE_CHECKING, Any, Callable + +from docutils import nodes +from docutils.parsers.rst import directives +from packaging.specifiers import InvalidSpecifier, SpecifierSet +from packaging.version import Version + +import sphinx +from sphinx.builders import Builder +from sphinx.locale import __ +from sphinx.util import logging +from sphinx.util.console import bold # type: ignore[attr-defined] +from sphinx.util.docutils import SphinxDirective +from sphinx.util.osutil import relpath + +if TYPE_CHECKING: + from collections.abc import Iterable, Sequence + + from docutils.nodes import Element, Node, TextElement + + from sphinx.application import Sphinx + from sphinx.util.typing import OptionSpec + + +logger = logging.getLogger(__name__) + +blankline_re = re.compile(r'^\s*<BLANKLINE>', re.MULTILINE) +doctestopt_re = re.compile(r'#\s*doctest:.+$', re.MULTILINE) + + +def is_allowed_version(spec: str, version: str) -> bool: + """Check `spec` satisfies `version` or not. + + This obeys PEP-440 specifiers: + https://peps.python.org/pep-0440/#version-specifiers + + Some examples: + + >>> is_allowed_version('<=3.5', '3.3') + True + >>> is_allowed_version('<=3.2', '3.3') + False + >>> is_allowed_version('>3.2, <4.0', '3.3') + True + """ + return Version(version) in SpecifierSet(spec) + + +# set up the necessary directives + +class TestDirective(SphinxDirective): + """ + Base class for doctest-related directives. + """ + + has_content = True + required_arguments = 0 + optional_arguments = 1 + final_argument_whitespace = True + + def run(self) -> list[Node]: + # use ordinary docutils nodes for test code: they get special attributes + # so that our builder recognizes them, and the other builders are happy. + code = '\n'.join(self.content) + test = None + if self.name == 'doctest': + if '<BLANKLINE>' in code: + # convert <BLANKLINE>s to ordinary blank lines for presentation + test = code + code = blankline_re.sub('', code) + if doctestopt_re.search(code) and 'no-trim-doctest-flags' not in self.options: + if not test: + test = code + code = doctestopt_re.sub('', code) + nodetype: type[TextElement] = nodes.literal_block + if self.name in ('testsetup', 'testcleanup') or 'hide' in self.options: + nodetype = nodes.comment + if self.arguments: + groups = [x.strip() for x in self.arguments[0].split(',')] + else: + groups = ['default'] + node = nodetype(code, code, testnodetype=self.name, groups=groups) + self.set_source_info(node) + if test is not None: + # only save if it differs from code + node['test'] = test + if self.name == 'doctest': + node['language'] = 'pycon' + elif self.name == 'testcode': + node['language'] = 'python' + elif self.name == 'testoutput': + # don't try to highlight output + node['language'] = 'none' + node['options'] = {} + if self.name in ('doctest', 'testoutput') and 'options' in self.options: + # parse doctest-like output comparison flags + option_strings = self.options['options'].replace(',', ' ').split() + for option in option_strings: + prefix, option_name = option[0], option[1:] + if prefix not in '+-': + self.state.document.reporter.warning( + __("missing '+' or '-' in '%s' option.") % option, + line=self.lineno) + continue + if option_name not in doctest.OPTIONFLAGS_BY_NAME: + self.state.document.reporter.warning( + __("'%s' is not a valid option.") % option_name, + line=self.lineno) + continue + flag = doctest.OPTIONFLAGS_BY_NAME[option[1:]] + node['options'][flag] = (option[0] == '+') + if self.name == 'doctest' and 'pyversion' in self.options: + try: + spec = self.options['pyversion'] + python_version = '.'.join([str(v) for v in sys.version_info[:3]]) + if not is_allowed_version(spec, python_version): + flag = doctest.OPTIONFLAGS_BY_NAME['SKIP'] + node['options'][flag] = True # Skip the test + except InvalidSpecifier: + self.state.document.reporter.warning( + __("'%s' is not a valid pyversion option") % spec, + line=self.lineno) + if 'skipif' in self.options: + node['skipif'] = self.options['skipif'] + if 'trim-doctest-flags' in self.options: + node['trim_flags'] = True + elif 'no-trim-doctest-flags' in self.options: + node['trim_flags'] = False + return [node] + + +class TestsetupDirective(TestDirective): + option_spec: OptionSpec = { + 'skipif': directives.unchanged_required, + } + + +class TestcleanupDirective(TestDirective): + option_spec: OptionSpec = { + 'skipif': directives.unchanged_required, + } + + +class DoctestDirective(TestDirective): + option_spec: OptionSpec = { + 'hide': directives.flag, + 'no-trim-doctest-flags': directives.flag, + 'options': directives.unchanged, + 'pyversion': directives.unchanged_required, + 'skipif': directives.unchanged_required, + 'trim-doctest-flags': directives.flag, + } + + +class TestcodeDirective(TestDirective): + option_spec: OptionSpec = { + 'hide': directives.flag, + 'no-trim-doctest-flags': directives.flag, + 'pyversion': directives.unchanged_required, + 'skipif': directives.unchanged_required, + 'trim-doctest-flags': directives.flag, + } + + +class TestoutputDirective(TestDirective): + option_spec: OptionSpec = { + 'hide': directives.flag, + 'no-trim-doctest-flags': directives.flag, + 'options': directives.unchanged, + 'pyversion': directives.unchanged_required, + 'skipif': directives.unchanged_required, + 'trim-doctest-flags': directives.flag, + } + + +parser = doctest.DocTestParser() + + +# helper classes + +class TestGroup: + def __init__(self, name: str) -> None: + self.name = name + self.setup: list[TestCode] = [] + self.tests: list[list[TestCode] | tuple[TestCode, None]] = [] + self.cleanup: list[TestCode] = [] + + def add_code(self, code: TestCode, prepend: bool = False) -> None: + if code.type == 'testsetup': + if prepend: + self.setup.insert(0, code) + else: + self.setup.append(code) + elif code.type == 'testcleanup': + self.cleanup.append(code) + elif code.type == 'doctest': + self.tests.append([code]) + elif code.type == 'testcode': + # "testoutput" may replace the second element + self.tests.append((code, None)) + elif code.type == 'testoutput': + if self.tests: + latest_test = self.tests[-1] + if len(latest_test) == 2: + self.tests[-1] = [latest_test[0], code] + else: + raise RuntimeError(__('invalid TestCode type')) + + def __repr__(self) -> str: + return (f'TestGroup(name={self.name!r}, setup={self.setup!r}, ' + f'cleanup={self.cleanup!r}, tests={self.tests!r})') + + +class TestCode: + def __init__(self, code: str, type: str, filename: str, + lineno: int, options: dict | None = None) -> None: + self.code = code + self.type = type + self.filename = filename + self.lineno = lineno + self.options = options or {} + + def __repr__(self) -> str: + return (f'TestCode({self.code!r}, {self.type!r}, filename={self.filename!r}, ' + f'lineno={self.lineno!r}, options={self.options!r})') + + +class SphinxDocTestRunner(doctest.DocTestRunner): + def summarize(self, out: Callable, verbose: bool | None = None, # type: ignore[override] + ) -> tuple[int, int]: + string_io = StringIO() + old_stdout = sys.stdout + sys.stdout = string_io + try: + res = super().summarize(verbose) + finally: + sys.stdout = old_stdout + out(string_io.getvalue()) + return res + + def _DocTestRunner__patched_linecache_getlines(self, filename: str, + module_globals: Any = None) -> Any: + # this is overridden from DocTestRunner adding the try-except below + m = self._DocTestRunner__LINECACHE_FILENAME_RE.match( # type: ignore[attr-defined] + filename) + if m and m.group('name') == self.test.name: + try: + example = self.test.examples[int(m.group('examplenum'))] + # because we compile multiple doctest blocks with the same name + # (viz. the group name) this might, for outer stack frames in a + # traceback, get the wrong test which might not have enough examples + except IndexError: + pass + else: + return example.source.splitlines(True) + return self.save_linecache_getlines( # type: ignore[attr-defined] + filename, module_globals) + + +# the new builder -- use sphinx-build.py -b doctest to run + +class DocTestBuilder(Builder): + """ + Runs test snippets in the documentation. + """ + name = 'doctest' + epilog = __('Testing of doctests in the sources finished, look at the ' + 'results in %(outdir)s/output.txt.') + + def init(self) -> None: + # default options + self.opt = self.config.doctest_default_flags + + # HACK HACK HACK + # doctest compiles its snippets with type 'single'. That is nice + # for doctest examples but unusable for multi-statement code such + # as setup code -- to be able to use doctest error reporting with + # that code nevertheless, we monkey-patch the "compile" it uses. + doctest.compile = self.compile # type: ignore[attr-defined] + + sys.path[0:0] = self.config.doctest_path + + self.type = 'single' + + self.total_failures = 0 + self.total_tries = 0 + self.setup_failures = 0 + self.setup_tries = 0 + self.cleanup_failures = 0 + self.cleanup_tries = 0 + + date = time.strftime('%Y-%m-%d %H:%M:%S') + + outpath = self.outdir.joinpath('output.txt') + self.outfile = outpath.open('w', encoding='utf-8') # NoQA: SIM115 + self.outfile.write(('Results of doctest builder run on %s\n' + '==================================%s\n') % + (date, '=' * len(date))) + + def _out(self, text: str) -> None: + logger.info(text, nonl=True) + self.outfile.write(text) + + def _warn_out(self, text: str) -> None: + if self.app.quiet or self.app.warningiserror: + logger.warning(text) + else: + logger.info(text, nonl=True) + self.outfile.write(text) + + def get_target_uri(self, docname: str, typ: str | None = None) -> str: + return '' + + def get_outdated_docs(self) -> set[str]: + return self.env.found_docs + + def finish(self) -> None: + # write executive summary + def s(v: int) -> str: + return 's' if v != 1 else '' + repl = (self.total_tries, s(self.total_tries), + self.total_failures, s(self.total_failures), + self.setup_failures, s(self.setup_failures), + self.cleanup_failures, s(self.cleanup_failures)) + self._out(''' +Doctest summary +=============== +%5d test%s +%5d failure%s in tests +%5d failure%s in setup code +%5d failure%s in cleanup code +''' % repl) + self.outfile.close() + + if self.total_failures or self.setup_failures or self.cleanup_failures: + self.app.statuscode = 1 + + def write(self, build_docnames: Iterable[str] | None, updated_docnames: Sequence[str], + method: str = 'update') -> None: + if build_docnames is None: + build_docnames = sorted(self.env.all_docs) + + logger.info(bold('running tests...')) + for docname in build_docnames: + # no need to resolve the doctree + doctree = self.env.get_doctree(docname) + self.test_doc(docname, doctree) + + def get_filename_for_node(self, node: Node, docname: str) -> str: + """Try to get the file which actually contains the doctest, not the + filename of the document it's included in.""" + try: + filename = relpath(node.source, self.env.srcdir)\ + .rsplit(':docstring of ', maxsplit=1)[0] + except Exception: + filename = self.env.doc2path(docname, False) + return filename + + @staticmethod + def get_line_number(node: Node) -> int: + """Get the real line number or admit we don't know.""" + # TODO: Work out how to store or calculate real (file-relative) + # line numbers for doctest blocks in docstrings. + if ':docstring of ' in path.basename(node.source or ''): + # The line number is given relative to the stripped docstring, + # not the file. This is correct where it is set, in + # `docutils.nodes.Node.setup_child`, but Sphinx should report + # relative to the file, not the docstring. + return None # type: ignore[return-value] + if node.line is not None: + # TODO: find the root cause of this off by one error. + return node.line - 1 + return None + + def skipped(self, node: Element) -> bool: + if 'skipif' not in node: + return False + else: + condition = node['skipif'] + context: dict[str, Any] = {} + if self.config.doctest_global_setup: + exec(self.config.doctest_global_setup, context) # NoQA: S102 + should_skip = eval(condition, context) # NoQA: PGH001 + if self.config.doctest_global_cleanup: + exec(self.config.doctest_global_cleanup, context) # NoQA: S102 + return should_skip + + def test_doc(self, docname: str, doctree: Node) -> None: + groups: dict[str, TestGroup] = {} + add_to_all_groups = [] + self.setup_runner = SphinxDocTestRunner(verbose=False, + optionflags=self.opt) + self.test_runner = SphinxDocTestRunner(verbose=False, + optionflags=self.opt) + self.cleanup_runner = SphinxDocTestRunner(verbose=False, + optionflags=self.opt) + + self.test_runner._fakeout = self.setup_runner._fakeout # type: ignore[attr-defined] + self.cleanup_runner._fakeout = self.setup_runner._fakeout # type: ignore[attr-defined] + + if self.config.doctest_test_doctest_blocks: + def condition(node: Node) -> bool: + return (isinstance(node, (nodes.literal_block, nodes.comment)) and + 'testnodetype' in node) or \ + isinstance(node, nodes.doctest_block) + else: + def condition(node: Node) -> bool: + return isinstance(node, (nodes.literal_block, nodes.comment)) \ + and 'testnodetype' in node + for node in doctree.findall(condition): # type: Element + if self.skipped(node): + continue + + source = node['test'] if 'test' in node else node.astext() + filename = self.get_filename_for_node(node, docname) + line_number = self.get_line_number(node) + if not source: + logger.warning(__('no code/output in %s block at %s:%s'), + node.get('testnodetype', 'doctest'), + filename, line_number) + code = TestCode(source, type=node.get('testnodetype', 'doctest'), + filename=filename, lineno=line_number, + options=node.get('options')) + node_groups = node.get('groups', ['default']) + if '*' in node_groups: + add_to_all_groups.append(code) + continue + for groupname in node_groups: + if groupname not in groups: + groups[groupname] = TestGroup(groupname) + groups[groupname].add_code(code) + for code in add_to_all_groups: + for group in groups.values(): + group.add_code(code) + if self.config.doctest_global_setup: + code = TestCode(self.config.doctest_global_setup, + 'testsetup', filename='<global_setup>', lineno=0) + for group in groups.values(): + group.add_code(code, prepend=True) + if self.config.doctest_global_cleanup: + code = TestCode(self.config.doctest_global_cleanup, + 'testcleanup', filename='<global_cleanup>', lineno=0) + for group in groups.values(): + group.add_code(code) + if not groups: + return + + show_successes = self.config.doctest_show_successes + if show_successes: + self._out('\n' + f'Document: {docname}\n' + f'----------{"-" * len(docname)}\n') + for group in groups.values(): + self.test_group(group) + # Separately count results from setup code + res_f, res_t = self.setup_runner.summarize(self._out, verbose=False) + self.setup_failures += res_f + self.setup_tries += res_t + if self.test_runner.tries: + res_f, res_t = self.test_runner.summarize( + self._out, verbose=show_successes) + self.total_failures += res_f + self.total_tries += res_t + if self.cleanup_runner.tries: + res_f, res_t = self.cleanup_runner.summarize( + self._out, verbose=show_successes) + self.cleanup_failures += res_f + self.cleanup_tries += res_t + + def compile(self, code: str, name: str, type: str, flags: Any, dont_inherit: bool) -> Any: + return compile(code, name, self.type, flags, dont_inherit) + + def test_group(self, group: TestGroup) -> None: + ns: dict = {} + + def run_setup_cleanup(runner: Any, testcodes: list[TestCode], what: Any) -> bool: + examples = [] + for testcode in testcodes: + example = doctest.Example(testcode.code, '', lineno=testcode.lineno) + examples.append(example) + if not examples: + return True + # simulate a doctest with the code + sim_doctest = doctest.DocTest(examples, {}, + f'{group.name} ({what} code)', + testcodes[0].filename, 0, None) + sim_doctest.globs = ns + old_f = runner.failures + self.type = 'exec' # the snippet may contain multiple statements + runner.run(sim_doctest, out=self._warn_out, clear_globs=False) + if runner.failures > old_f: + return False + return True + + # run the setup code + if not run_setup_cleanup(self.setup_runner, group.setup, 'setup'): + # if setup failed, don't run the group + return + + # run the tests + for code in group.tests: + if len(code) == 1: + # ordinary doctests (code/output interleaved) + try: + test = parser.get_doctest(code[0].code, {}, group.name, + code[0].filename, code[0].lineno) + except Exception: + logger.warning(__('ignoring invalid doctest code: %r'), code[0].code, + location=(code[0].filename, code[0].lineno)) + continue + if not test.examples: + continue + for example in test.examples: + # apply directive's comparison options + new_opt = code[0].options.copy() + new_opt.update(example.options) + example.options = new_opt + self.type = 'single' # as for ordinary doctests + else: + # testcode and output separate + output = code[1].code if code[1] else '' + options = code[1].options if code[1] else {} + # disable <BLANKLINE> processing as it is not needed + options[doctest.DONT_ACCEPT_BLANKLINE] = True + # find out if we're testing an exception + m = parser._EXCEPTION_RE.match(output) # type: ignore[attr-defined] + if m: + exc_msg = m.group('msg') + else: + exc_msg = None + example = doctest.Example(code[0].code, output, exc_msg=exc_msg, + lineno=code[0].lineno, options=options) + test = doctest.DocTest([example], {}, group.name, + code[0].filename, code[0].lineno, None) + self.type = 'exec' # multiple statements again + # DocTest.__init__ copies the globs namespace, which we don't want + test.globs = ns + # also don't clear the globs namespace after running the doctest + self.test_runner.run(test, out=self._warn_out, clear_globs=False) + + # run the cleanup + run_setup_cleanup(self.cleanup_runner, group.cleanup, 'cleanup') + + +def setup(app: Sphinx) -> dict[str, Any]: + app.add_directive('testsetup', TestsetupDirective) + app.add_directive('testcleanup', TestcleanupDirective) + app.add_directive('doctest', DoctestDirective) + app.add_directive('testcode', TestcodeDirective) + app.add_directive('testoutput', TestoutputDirective) + app.add_builder(DocTestBuilder) + # this config value adds to sys.path + app.add_config_value('doctest_show_successes', True, False, (bool,)) + app.add_config_value('doctest_path', [], False) + app.add_config_value('doctest_test_doctest_blocks', 'default', False) + app.add_config_value('doctest_global_setup', '', False) + app.add_config_value('doctest_global_cleanup', '', False) + app.add_config_value( + 'doctest_default_flags', + doctest.DONT_ACCEPT_TRUE_FOR_1 | doctest.ELLIPSIS | doctest.IGNORE_EXCEPTION_DETAIL, + False) + return {'version': sphinx.__display_version__, 'parallel_read_safe': True} |