"""Test the HTML builder and check output against XPath."""
import hashlib
import os
import posixpath
import re
from itertools import chain, cycle
from pathlib import Path
from unittest.mock import ANY, call, patch
import pytest
from html5lib import HTMLParser
import sphinx.builders.html
from sphinx.builders.html import validate_html_extra_path, validate_html_static_path
from sphinx.builders.html._assets import _file_checksum
from sphinx.errors import ConfigError, ThemeError
from sphinx.testing.util import strip_escseq
from sphinx.util.inventory import InventoryFile
FIGURE_CAPTION = ".//figure/figcaption/p"
ENV_WARNINGS = """\
%(root)s/autodoc_fodder.py:docstring of autodoc_fodder.MarkupError:\\d+: \
WARNING: Explicit markup ends without a blank line; unexpected unindent.
%(root)s/index.rst:\\d+: WARNING: Encoding 'utf-8-sig' used for reading included \
file '%(root)s/wrongenc.inc' seems to be wrong, try giving an :encoding: option
%(root)s/index.rst:\\d+: WARNING: invalid single index entry ''
%(root)s/index.rst:\\d+: WARNING: image file not readable: foo.png
%(root)s/index.rst:\\d+: WARNING: download file not readable: %(root)s/nonexisting.png
%(root)s/undecodable.rst:\\d+: WARNING: undecodable source characters, replacing \
with "\\?": b?'here: >>>(\\\\|/)xbb<<<((\\\\|/)r)?'
"""
HTML_WARNINGS = ENV_WARNINGS + """\
%(root)s/index.rst:\\d+: WARNING: unknown option: '&option'
%(root)s/index.rst:\\d+: WARNING: citation not found: missing
%(root)s/index.rst:\\d+: WARNING: a suitable image for html builder not found: foo.\\*
%(root)s/index.rst:\\d+: WARNING: Lexing literal_block ".*" as "c" resulted in an error at token: ".*". Retrying in relaxed mode.
"""
etree_cache = {}
@pytest.fixture(scope='module')
def cached_etree_parse():
def parse(fname):
if fname in etree_cache:
return etree_cache[fname]
with (fname).open('rb') as fp:
etree = HTMLParser(namespaceHTMLElements=False).parse(fp)
etree_cache.clear()
etree_cache[fname] = etree
return etree
yield parse
etree_cache.clear()
def flat_dict(d):
return chain.from_iterable(
[
zip(cycle([fname]), values)
for fname, values in d.items()
],
)
def tail_check(check):
rex = re.compile(check)
def checker(nodes):
for node in nodes:
if node.tail and rex.search(node.tail):
return True
msg = f'{check!r} not found in tail of any nodes {nodes}'
raise AssertionError(msg)
return checker
def check_xpath(etree, fname, path, check, be_found=True):
nodes = list(etree.findall(path))
if check is None:
assert nodes == [], ('found any nodes matching xpath '
'%r in file %s' % (path, fname))
return
else:
assert nodes != [], ('did not find any node matching xpath '
'%r in file %s' % (path, fname))
if callable(check):
check(nodes)
elif not check:
# only check for node presence
pass
else:
def get_text(node):
if node.text is not None:
# the node has only one text
return node.text
else:
# the node has tags and text; gather texts just under the node
return ''.join(n.tail or '' for n in node)
rex = re.compile(check)
if be_found:
if any(rex.search(get_text(node)) for node in nodes):
return
else:
if all(not rex.search(get_text(node)) for node in nodes):
return
raise AssertionError('%r not found in any node matching '
'path %s in %s: %r' % (check, path, fname,
[node.text for node in nodes]))
@pytest.mark.sphinx('html', testroot='warnings')
def test_html_warnings(app, warning):
app.build()
html_warnings = strip_escseq(re.sub(re.escape(os.sep) + '{1,2}', '/', warning.getvalue()))
html_warnings_exp = HTML_WARNINGS % {
'root': re.escape(app.srcdir.as_posix())}
assert re.match(html_warnings_exp + '$', html_warnings), \
"Warnings don't match:\n" + \
'--- Expected (regex):\n' + html_warnings_exp + \
'--- Got:\n' + html_warnings
def test_html4_error(make_app, tmp_path):
(tmp_path / 'conf.py').write_text('', encoding='utf-8')
with pytest.raises(
ConfigError,
match='HTML 4 is no longer supported by Sphinx',
):
make_app(
buildername='html',
srcdir=tmp_path,
confoverrides={'html4_writer': True},
)
@pytest.mark.parametrize(("fname", "expect"), flat_dict({
'images.html': [
(".//img[@src='_images/img.png']", ''),
(".//img[@src='_images/img1.png']", ''),
(".//img[@src='_images/simg.png']", ''),
(".//img[@src='_images/svgimg.svg']", ''),
(".//a[@href='_sources/images.txt']", ''),
],
'subdir/images.html': [
(".//img[@src='../_images/img1.png']", ''),
(".//img[@src='../_images/rimg.png']", ''),
],
'subdir/includes.html': [
(".//a[@class='reference download internal']", ''),
(".//img[@src='../_images/img.png']", ''),
(".//p", 'This is an include file.'),
(".//pre/span", 'line 1'),
(".//pre/span", 'line 2'),
],
'includes.html': [
(".//pre", 'Max Strauß'),
(".//a[@class='reference download internal']", ''),
(".//pre/span", '"quotes"'),
(".//pre/span", "'included'"),
(".//pre/span[@class='s2']", 'üöä'),
(".//div[@class='inc-pyobj1 highlight-text notranslate']//pre",
r'^class Foo:\n pass\n\s*$'),
(".//div[@class='inc-pyobj2 highlight-text notranslate']//pre",
r'^ def baz\(\):\n pass\n\s*$'),
(".//div[@class='inc-lines highlight-text notranslate']//pre",
r'^class Foo:\n pass\nclass Bar:\n$'),
(".//div[@class='inc-startend highlight-text notranslate']//pre",
'^foo = "Including Unicode characters: üöä"\\n$'),
(".//div[@class='inc-preappend highlight-text notranslate']//pre",
r'(?m)^START CODE$'),
(".//div[@class='inc-pyobj-dedent highlight-python notranslate']//span",
r'def'),
(".//div[@class='inc-tab3 highlight-text notranslate']//pre",
r'-| |-'),
(".//div[@class='inc-tab8 highlight-python notranslate']//pre/span",
r'-| |-'),
],
'autodoc.html': [
(".//dl[@class='py class']/dt[@id='autodoc_target.Class']", ''),
(".//dl[@class='py function']/dt[@id='autodoc_target.function']/em/span/span", r'\*\*'),
(".//dl[@class='py function']/dt[@id='autodoc_target.function']/em/span/span", r'kwds'),
(".//dd/p", r'Return spam\.'),
],
'extapi.html': [
(".//strong", 'from class: Bar'),
],
'markup.html': [
(".//title", 'set by title directive'),
(".//p/em", 'Section author: Georg Brandl'),
(".//p/em", 'Module author: Georg Brandl'),
# created by the meta directive
(".//meta[@name='author'][@content='Me']", ''),
(".//meta[@name='keywords'][@content='docs, sphinx']", ''),
# a label created by ``.. _label:``
(".//div[@id='label']", ''),
# code with standard code blocks
(".//pre", '^some code$'),
# an option list
(".//span[@class='option']", '--help'),
# admonitions
(".//p[@class='admonition-title']", 'My Admonition'),
(".//div[@class='admonition note']/p", 'Note text.'),
(".//div[@class='admonition warning']/p", 'Warning text.'),
# inline markup
(".//li/p/strong", r'^command\\n$'),
(".//li/p/strong", r'^program\\n$'),
(".//li/p/em", r'^dfn\\n$'),
(".//li/p/kbd", r'^kbd\\n$'),
(".//li/p/span", 'File \N{TRIANGULAR BULLET} Close'),
(".//li/p/code/span[@class='pre']", '^a/$'),
(".//li/p/code/em/span[@class='pre']", '^varpart$'),
(".//li/p/code/em/span[@class='pre']", '^i$'),
(".//a[@href='https://peps.python.org/pep-0008/']"
"[@class='pep reference external']/strong", 'PEP 8'),
(".//a[@href='https://peps.python.org/pep-0008/']"
"[@class='pep reference external']/strong",
'Python Enhancement Proposal #8'),
(".//a[@href='https://datatracker.ietf.org/doc/html/rfc1.html']"
"[@class='rfc reference external']/strong", 'RFC 1'),
(".//a[@href='https://datatracker.ietf.org/doc/html/rfc1.html']"
"[@class='rfc reference external']/strong", 'Request for Comments #1'),
(".//a[@href='objects.html#envvar-HOME']"
"[@class='reference internal']/code/span[@class='pre']", 'HOME'),
(".//a[@href='#with']"
"[@class='reference internal']/code/span[@class='pre']", '^with$'),
(".//a[@href='#grammar-token-try_stmt']"
"[@class='reference internal']/code/span", '^statement$'),
(".//a[@href='#some-label'][@class='reference internal']/span", '^here$'),
(".//a[@href='#some-label'][@class='reference internal']/span", '^there$'),
(".//a[@href='subdir/includes.html']"
"[@class='reference internal']/span", 'Including in subdir'),
(".//a[@href='objects.html#cmdoption-python-c']"
"[@class='reference internal']/code/span[@class='pre']", '-c'),
# abbreviations
(".//abbr[@title='abbreviation']", '^abbr$'),
# version stuff
(".//div[@class='versionadded']/p/span", 'New in version 0.6: '),
(".//div[@class='versionadded']/p/span",
tail_check('First paragraph of versionadded')),
(".//div[@class='versionchanged']/p/span",
tail_check('First paragraph of versionchanged')),
(".//div[@class='versionchanged']/p",
'Second paragraph of versionchanged'),
# footnote reference
(".//a[@class='footnote-reference brackets']", r'1'),
# created by reference lookup
(".//a[@href='index.html#ref1']", ''),
# ``seealso`` directive
(".//div/p[@class='admonition-title']", 'See also'),
# a ``hlist`` directive
(".//table[@class='hlist']/tbody/tr/td/ul/li/p", '^This$'),
# a ``centered`` directive
(".//p[@class='centered']/strong", 'LICENSE'),
# a glossary
(".//dl/dt[@id='term-boson']", 'boson'),
(".//dl/dt[@id='term-boson']/a", '¶'),
# a production list
(".//pre/strong", 'try_stmt'),
(".//pre/a[@href='#grammar-token-try1_stmt']/code/span", 'try1_stmt'),
# tests for ``only`` directive
(".//p", 'A global substitution!'),
(".//p", 'In HTML.'),
(".//p", 'In both.'),
(".//p", 'Always present'),
# tests for ``any`` role
(".//a[@href='#with']/span", 'headings'),
(".//a[@href='objects.html#func_without_body']/code/span", 'objects'),
# tests for numeric labels
(".//a[@href='#id1'][@class='reference internal']/span", 'Testing various markup'),
# tests for smartypants
(".//li/p", 'Smart “quotes” in English ‘text’.'),
(".//li/p", 'Smart — long and – short dashes.'),
(".//li/p", 'Ellipsis…'),
(".//li/p/code/span[@class='pre']", 'foo--"bar"...'),
(".//p", 'Этот «абзац» должен использовать „русские“ кавычки.'),
(".//p", 'Il dit : « C’est “super” ! »'),
],
'objects.html': [
(".//dt[@id='mod.Cls.meth1']", ''),
(".//dt[@id='errmod.Error']", ''),
(".//dt/span[@class='sig-name descname']/span[@class='pre']", r'long\(parameter,'),
(".//dt/span[@class='sig-name descname']/span[@class='pre']", r'list\)'),
(".//dt/span[@class='sig-name descname']/span[@class='pre']", 'another'),
(".//dt/span[@class='sig-name descname']/span[@class='pre']", 'one'),
(".//a[@href='#mod.Cls'][@class='reference internal']", ''),
(".//dl[@class='std userdesc']", ''),
(".//dt[@id='userdesc-myobj']", ''),
(".//a[@href='#userdesc-myobj'][@class='reference internal']", ''),
# docfields
(".//a[@class='reference internal'][@href='#TimeInt']/em", 'TimeInt'),
(".//a[@class='reference internal'][@href='#Time']", 'Time'),
(".//a[@class='reference internal'][@href='#errmod.Error']/strong", 'Error'),
# C references
(".//span[@class='pre']", 'CFunction()'),
(".//a[@href='#c.Sphinx_DoSomething']", ''),
(".//a[@href='#c.SphinxStruct.member']", ''),
(".//a[@href='#c.SPHINX_USE_PYTHON']", ''),
(".//a[@href='#c.SphinxType']", ''),
(".//a[@href='#c.sphinx_global']", ''),
# test global TOC created by toctree()
(".//ul[@class='current']/li[@class='toctree-l1 current']/a[@href='#']",
'Testing object descriptions'),
(".//li[@class='toctree-l1']/a[@href='markup.html']",
'Testing various markup'),
# test unknown field names
(".//dt[@class='field-odd']", 'Field_name'),
(".//dt[@class='field-even']", 'Field_name all lower'),
(".//dt[@class='field-odd']", 'FIELD_NAME'),
(".//dt[@class='field-even']", 'FIELD_NAME ALL CAPS'),
(".//dt[@class='field-odd']", 'Field_Name'),
(".//dt[@class='field-even']", 'Field_Name All Word Caps'),
(".//dt[@class='field-odd']", 'Field_name'),
(".//dt[@class='field-even']", 'Field_name First word cap'),
(".//dt[@class='field-odd']", 'FIELd_name'),
(".//dt[@class='field-even']", 'FIELd_name PARTial caps'),
# custom sidebar
(".//h4", 'Custom sidebar'),
# docfields
(".//dd[@class='field-odd']/p/strong", '^moo$'),
(".//dd[@class='field-odd']/p/strong", tail_check(r'\(Moo\) .* Moo')),
(".//dd[@class='field-odd']/ul/li/p/strong", '^hour$'),
(".//dd[@class='field-odd']/ul/li/p/em", '^DuplicateType$'),
(".//dd[@class='field-odd']/ul/li/p/em", tail_check(r'.* Some parameter')),
# others
(".//a[@class='reference internal'][@href='#cmdoption-perl-arg-p']/code/span",
'perl'),
(".//a[@class='reference internal'][@href='#cmdoption-perl-arg-p']/code/span",
'\\+p'),
(".//a[@class='reference internal'][@href='#cmdoption-perl-ObjC']/code/span",
'--ObjC\\+\\+'),
(".//a[@class='reference internal'][@href='#cmdoption-perl-plugin.option']/code/span",
'--plugin.option'),
(".//a[@class='reference internal'][@href='#cmdoption-perl-arg-create-auth-token']"
"/code/span",
'create-auth-token'),
(".//a[@class='reference internal'][@href='#cmdoption-perl-arg-arg']/code/span",
'arg'),
(".//a[@class='reference internal'][@href='#cmdoption-perl-j']/code/span",
'-j'),
(".//a[@class='reference internal'][@href='#cmdoption-hg-arg-commit']/code/span",
'hg'),
(".//a[@class='reference internal'][@href='#cmdoption-hg-arg-commit']/code/span",
'commit'),
(".//a[@class='reference internal'][@href='#cmdoption-git-commit-p']/code/span",
'git'),
(".//a[@class='reference internal'][@href='#cmdoption-git-commit-p']/code/span",
'commit'),
(".//a[@class='reference internal'][@href='#cmdoption-git-commit-p']/code/span",
'-p'),
],
'index.html': [
(".//meta[@name='hc'][@content='hcval']", ''),
(".//meta[@name='hc_co'][@content='hcval_co']", ''),
(".//li[@class='toctree-l1']/a", 'Testing various markup'),
(".//li[@class='toctree-l2']/a", 'Inline markup'),
(".//title", 'Sphinx '),
(".//div[@class='footer']", 'copyright text credits'),
(".//a[@href='http://python.org/']"
"[@class='reference external']", ''),
(".//li/p/a[@href='genindex.html']/span", 'Index'),
(".//li/p/a[@href='py-modindex.html']/span", 'Module Index'),
# custom sidebar only for contents
(".//h4", 'Contents sidebar'),
# custom JavaScript
(".//script[@src='file://moo.js']", ''),
# URL in contents
(".//a[@class='reference external'][@href='http://sphinx-doc.org/']",
'http://sphinx-doc.org/'),
(".//a[@class='reference external'][@href='http://sphinx-doc.org/latest/']",
'Latest reference'),
# Indirect hyperlink targets across files
(".//a[@href='markup.html#some-label'][@class='reference internal']/span",
'^indirect hyperref$'),
],
'bom.html': [
(".//title", " File with UTF-8 BOM"),
],
'extensions.html': [
(".//a[@href='http://python.org/dev/']", "http://python.org/dev/"),
(".//a[@href='http://bugs.python.org/issue1000']", "issue 1000"),
(".//a[@href='http://bugs.python.org/issue1042']", "explicit caption"),
],
'genindex.html': [
# index entries
(".//a/strong", "Main"),
(".//a/strong", "[1]"),
(".//a/strong", "Other"),
(".//a", "entry"),
(".//li/a", "double"),
],
'otherext.html': [
(".//h1", "Generated section"),
(".//a[@href='_sources/otherext.foo.txt']", ''),
],
}))
@pytest.mark.sphinx('html', tags=['testtag'],
confoverrides={'html_context.hckey_co': 'hcval_co'})
@pytest.mark.test_params(shared_result='test_build_html_output')
def test_html5_output(app, cached_etree_parse, fname, expect):
app.build()
print(app.outdir / fname)
check_xpath(cached_etree_parse(app.outdir / fname), fname, *expect)
@pytest.mark.parametrize(("fname", "expect"), flat_dict({
'index.html': [
(".//div[@class='citation']/span", r'Ref1'),
(".//div[@class='citation']/span", r'Ref_1'),
],
'footnote.html': [
(".//a[@class='footnote-reference brackets'][@href='#id9'][@id='id1']", r"1"),
(".//a[@class='footnote-reference brackets'][@href='#id10'][@id='id2']", r"2"),
(".//a[@class='footnote-reference brackets'][@href='#foo'][@id='id3']", r"3"),
(".//a[@class='reference internal'][@href='#bar'][@id='id4']/span", r"\[bar\]"),
(".//a[@class='reference internal'][@href='#baz-qux'][@id='id5']/span", r"\[baz_qux\]"),
(".//a[@class='footnote-reference brackets'][@href='#id11'][@id='id6']", r"4"),
(".//a[@class='footnote-reference brackets'][@href='#id12'][@id='id7']", r"5"),
(".//aside[@class='footnote brackets']/span/a[@href='#id1']", r"1"),
(".//aside[@class='footnote brackets']/span/a[@href='#id2']", r"2"),
(".//aside[@class='footnote brackets']/span/a[@href='#id3']", r"3"),
(".//div[@class='citation']/span/a[@href='#id4']", r"bar"),
(".//div[@class='citation']/span/a[@href='#id5']", r"baz_qux"),
(".//aside[@class='footnote brackets']/span/a[@href='#id6']", r"4"),
(".//aside[@class='footnote brackets']/span/a[@href='#id7']", r"5"),
(".//aside[@class='footnote brackets']/span/a[@href='#id8']", r"6"),
],
}))
@pytest.mark.sphinx('html')
@pytest.mark.test_params(shared_result='test_build_html_output_docutils18')
def test_docutils_output(app, cached_etree_parse, fname, expect):
app.build()
print(app.outdir / fname)
check_xpath(cached_etree_parse(app.outdir / fname), fname, *expect)
@pytest.mark.sphinx('html', parallel=2)
def test_html_parallel(app):
app.build()
@pytest.mark.sphinx('html')
@pytest.mark.test_params(shared_result='test_build_html_output')
def test_html_download(app):
app.build()
# subdir/includes.html
result = (app.outdir / 'subdir' / 'includes.html').read_text(encoding='utf8')
pattern = ('')
matched = re.search(pattern, result)
assert matched
assert (app.outdir / matched.group(1)).exists()
filename = matched.group(1)
# includes.html
result = (app.outdir / 'includes.html').read_text(encoding='utf8')
pattern = (' ')
matched = re.search(pattern, result)
assert matched
assert (app.outdir / matched.group(1)).exists()
assert matched.group(1) == filename
pattern = (' ')
matched = re.search(pattern, result)
assert matched
assert (app.outdir / matched.group(1) / "file_with_special_#_chars.xyz").exists()
@pytest.mark.sphinx('html', testroot='roles-download')
def test_html_download_role(app, status, warning):
app.build()
digest = hashlib.md5(b'dummy.dat', usedforsecurity=False).hexdigest()
assert (app.outdir / '_downloads' / digest / 'dummy.dat').exists()
digest_another = hashlib.md5(b'another/dummy.dat', usedforsecurity=False).hexdigest()
assert (app.outdir / '_downloads' / digest_another / 'dummy.dat').exists()
content = (app.outdir / 'index.html').read_text(encoding='utf8')
assert ((''
''
'dummy.dat
'
''
'another/dummy.dat
'
'not_found.dat
'
''
'Sphinx logo '
'
\nThe caption of pic '
'
\n '
in content)
@pytest.mark.sphinx('html', testroot='directives-raw')
def test_html_raw_directive(app, status, warning):
app.builder.build_all()
result = (app.outdir / 'index.html').read_text(encoding='utf8')
# standard case
assert 'standalone raw directive (HTML)' in result
assert 'standalone raw directive (LaTeX)' not in result
# with substitution
assert 'HTML: abc def ghi
' in result
assert 'LaTeX: abc ghi
' in result
@pytest.mark.parametrize(("fname", "expect"), flat_dict({
'index.html': [
(".//link[@href='_static/persistent.css']"
"[@rel='stylesheet']", '', True),
(".//link[@href='_static/default.css']"
"[@rel='stylesheet']"
"[@title='Default']", '', True),
(".//link[@href='_static/alternate1.css']"
"[@rel='alternate stylesheet']"
"[@title='Alternate']", '', True),
(".//link[@href='_static/alternate2.css']"
"[@rel='alternate stylesheet']", '', True),
(".//link[@href='_static/more_persistent.css']"
"[@rel='stylesheet']", '', True),
(".//link[@href='_static/more_default.css']"
"[@rel='stylesheet']"
"[@title='Default']", '', True),
(".//link[@href='_static/more_alternate1.css']"
"[@rel='alternate stylesheet']"
"[@title='Alternate']", '', True),
(".//link[@href='_static/more_alternate2.css']"
"[@rel='alternate stylesheet']", '', True),
],
}))
@pytest.mark.sphinx('html', testroot='stylesheets')
def test_alternate_stylesheets(app, cached_etree_parse, fname, expect):
app.build()
check_xpath(cached_etree_parse(app.outdir / fname), fname, *expect)
@pytest.mark.sphinx('html', testroot='html_style')
def test_html_style(app, status, warning):
app.build()
result = (app.outdir / 'index.html').read_text(encoding='utf8')
assert ' 
' in result)
assert (app.outdir / '_images/img_#1.png').exists()
@pytest.mark.sphinx('html', testroot='remote-logo')
def test_html_remote_logo(app, status, warning):
app.builder.build_all()
result = (app.outdir / 'index.html').read_text(encoding='utf8')
assert ('
' in result)
assert ('' in result)
assert not (app.outdir / 'python-logo.png').exists()
@pytest.mark.sphinx('html', testroot='local-logo')
def test_html_local_logo(app, status, warning):
app.builder.build_all()
result = (app.outdir / 'index.html').read_text(encoding='utf8')
assert ('
' in result)
assert (app.outdir / '_static/img.png').exists()
@pytest.mark.sphinx('html', testroot='basic')
def test_html_sidebar(app, status, warning):
ctx = {}
# default for alabaster
app.builder.build_all()
result = (app.outdir / 'index.html').read_text(encoding='utf8')
assert ('
', context)
# scaled_image_link
assert re.search('\n'
'