Adding upstream version 7.2.6.upstream/7.2.6upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

6 files changed, 575 insertions, 0 deletions

diff --git a/utils/CHANGES_template b/utils/CHANGES_template
new file mode 100644
index 0000000..a655c46
--- /dev/null
+++ b/utils/CHANGES_template
@@ -0,0 +1,20 @@
+Release x.y.z (in development)
+==============================
+
+Dependencies
+------------
+
+Incompatible changes
+--------------------
+
+Deprecated
+----------
+
+Features added
+--------------
+
+Bugs fixed
+----------
+
+Testing
+-------
diff --git a/utils/__init__.py b/utils/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/utils/__init__.py
diff --git a/utils/babel_runner.py b/utils/babel_runner.py
new file mode 100644
index 0000000..dfb58db
--- /dev/null
+++ b/utils/babel_runner.py
@@ -0,0 +1,250 @@
+"""Run babel for translations.
+
+Usage:
+
+babel_runner.py extract
+    Extract messages from the source code and update the ".pot" template file.
+
+babel_runner.py update
+    Update all language catalogues in "sphinx/locale/<language>/LC_MESSAGES"
+    with the current messages in the template file.
+
+babel_runner.py compile
+    Compile the ".po" catalogue files to ".mo" and ".js" files.
+"""
+
+import json
+import logging
+import os
+import sys
+import tempfile
+
+from babel.messages.catalog import Catalog
+from babel.messages.extract import (
+    DEFAULT_KEYWORDS,
+    extract,
+    extract_javascript,
+    extract_python,
+)
+from babel.messages.mofile import write_mo
+from babel.messages.pofile import read_po, write_po
+from babel.util import pathmatch
+from jinja2.ext import babel_extract as extract_jinja2
+
+ROOT = os.path.realpath(os.path.join(os.path.abspath(__file__), "..", ".."))
+TEX_DELIMITERS = {
+    'variable_start_string': '<%=',
+    'variable_end_string': '%>',
+    'block_start_string': '<%',
+    'block_end_string': '%>',
+}
+METHOD_MAP = [
+    # Extraction from Python source files
+    ('**.py', extract_python),
+    # Extraction from Jinja2 template files
+    ('**/templates/latex/**.tex_t', extract_jinja2),
+    ('**/templates/latex/**.sty_t', extract_jinja2),
+    # Extraction from Jinja2 HTML templates
+    ('**/themes/**.html', extract_jinja2),
+    # Extraction from Jinja2 XML templates
+    ('**/themes/**.xml', extract_jinja2),
+    # Extraction from JavaScript files
+    ('**.js', extract_javascript),
+    ('**.js_t', extract_javascript),
+]
+OPTIONS_MAP = {
+    # Extraction from Python source files
+    '**.py': {
+        'encoding': 'utf-8',
+    },
+    # Extraction from Jinja2 template files
+    '**/templates/latex/**.tex_t': TEX_DELIMITERS.copy(),
+    '**/templates/latex/**.sty_t': TEX_DELIMITERS.copy(),
+    # Extraction from Jinja2 HTML templates
+    '**/themes/**.html': {
+        'encoding': 'utf-8',
+        'ignore_tags': 'script,style',
+        'include_attrs': 'alt title summary',
+    },
+}
+KEYWORDS = {**DEFAULT_KEYWORDS, '_': None, '__': None}
+
+
+def run_extract():
+    """Message extraction function."""
+    log = _get_logger()
+
+    with open('sphinx/__init__.py', encoding='utf-8') as f:
+        for line in f.read().splitlines():
+            if line.startswith('__version__ = '):
+                # remove prefix; strip whitespace; remove quotation marks
+                sphinx_version = line[14:].strip()[1:-1]
+                break
+
+    input_path = 'sphinx'
+    catalogue = Catalog(project='Sphinx', version=sphinx_version, charset='utf-8')
+
+    base = os.path.abspath(input_path)
+    for root, dirnames, filenames in os.walk(base):
+        relative_root = os.path.relpath(root, base) if root != base else ''
+        dirnames.sort()
+        for filename in sorted(filenames):
+            relative_name = os.path.join(relative_root, filename)
+            for pattern, method in METHOD_MAP:
+                if not pathmatch(pattern, relative_name):
+                    continue
+
+                options = {}
+                for opt_pattern, opt_dict in OPTIONS_MAP.items():
+                    if pathmatch(opt_pattern, relative_name):
+                        options = opt_dict
+                with open(os.path.join(root, filename), 'rb') as fileobj:
+                    for lineno, message, comments, context in extract(
+                        method, fileobj, KEYWORDS, options=options,
+                    ):
+                        filepath = os.path.join(input_path, relative_name)
+                        catalogue.add(
+                            message, None, [(filepath, lineno)],
+                            auto_comments=comments, context=context,
+                        )
+                break
+
+    output_file = os.path.join('sphinx', 'locale', 'sphinx.pot')
+    log.info('writing PO template file to %s', output_file)
+    with open(output_file, 'wb') as outfile:
+        write_po(outfile, catalogue)
+
+
+def run_update():
+    """Catalog merging command."""
+
+    log = _get_logger()
+
+    domain = 'sphinx'
+    locale_dir = os.path.join('sphinx', 'locale')
+    template_file = os.path.join(locale_dir, 'sphinx.pot')
+
+    with open(template_file, encoding='utf-8') as infile:
+        template = read_po(infile)
+
+    for locale in os.listdir(locale_dir):
+        filename = os.path.join(locale_dir, locale, 'LC_MESSAGES', f'{domain}.po')
+        if not os.path.exists(filename):
+            continue
+
+        log.info('updating catalog %s based on %s', filename, template_file)
+        with open(filename, encoding='utf-8') as infile:
+            catalog = read_po(infile, locale=locale, domain=domain)
+
+        catalog.update(template)
+        tmp_name = os.path.join(
+            os.path.dirname(filename), tempfile.gettempprefix() + os.path.basename(filename),
+        )
+        try:
+            with open(tmp_name, 'wb') as tmpfile:
+                write_po(tmpfile, catalog)
+        except Exception:
+            os.remove(tmp_name)
+            raise
+
+        os.replace(tmp_name, filename)
+
+
+def run_compile():
+    """
+    Catalog compilation command.
+
+    An extended command that writes all message strings that occur in
+    JavaScript files to a JavaScript file along with the .mo file.
+
+    Unfortunately, babel's setup command isn't built very extensible, so
+    most of the run() code is duplicated here.
+    """
+
+    log = _get_logger()
+
+    directory = os.path.join('sphinx', 'locale')
+    total_errors = 0
+
+    for locale in os.listdir(directory):
+        po_file = os.path.join(directory, locale, 'LC_MESSAGES', 'sphinx.po')
+        if not os.path.exists(po_file):
+            continue
+
+        with open(po_file, encoding='utf-8') as infile:
+            catalog = read_po(infile, locale)
+
+        if catalog.fuzzy:
+            log.info('catalog %s is marked as fuzzy, skipping', po_file)
+            continue
+
+        for message, errors in catalog.check():
+            for error in errors:
+                total_errors += 1
+                log.error('error: %s:%d: %s\nerror:     in message string: %s',
+                          po_file, message.lineno, error, message.string)
+
+        mo_file = os.path.join(directory, locale, 'LC_MESSAGES', 'sphinx.mo')
+        log.info('compiling catalog %s to %s', po_file, mo_file)
+        with open(mo_file, 'wb') as outfile:
+            write_mo(outfile, catalog, use_fuzzy=False)
+
+        js_file = os.path.join(directory, locale, 'LC_MESSAGES', 'sphinx.js')
+        log.info('writing JavaScript strings in catalog %s to %s', po_file, js_file)
+        js_catalogue = {}
+        for message in catalog:
+            if any(
+                    x[0].endswith(('.js', '.js.jinja', '.js_t', '.html'))
+                    for x in message.locations
+            ):
+                msgid = message.id
+                if isinstance(msgid, (list, tuple)):
+                    msgid = msgid[0]
+                js_catalogue[msgid] = message.string
+
+        obj = json.dumps({
+            'messages': js_catalogue,
+            'plural_expr': catalog.plural_expr,
+            'locale': str(catalog.locale),
+        }, sort_keys=True, indent=4)
+        with open(js_file, 'wb') as outfile:
+            # to ensure lines end with ``\n`` rather than ``\r\n``:
+            outfile.write(f'Documentation.addTranslations({obj});'.encode())
+
+    if total_errors > 0:
+        log.error('%d errors encountered.', total_errors)
+        print("Compiling failed.", file=sys.stderr)
+        raise SystemExit(2)
+
+
+def _get_logger():
+    log = logging.getLogger('babel')
+    log.setLevel(logging.INFO)
+    handler = logging.StreamHandler()
+    handler.setFormatter(logging.Formatter('%(message)s'))
+    log.addHandler(handler)
+    return log
+
+
+if __name__ == '__main__':
+    try:
+        action = sys.argv[1].lower()
+    except IndexError:
+        print(__doc__, file=sys.stderr)
+        raise SystemExit(2) from None
+
+    os.chdir(ROOT)
+    if action == "extract":
+        raise SystemExit(run_extract())
+    if action == "update":
+        raise SystemExit(run_update())
+    if action == "compile":
+        raise SystemExit(run_compile())
+    if action == "all":
+        exit_code = run_extract()
+        if exit_code:
+            raise SystemExit(exit_code)
+        exit_code = run_update()
+        if exit_code:
+            raise SystemExit(exit_code)
+        raise SystemExit(run_compile())
diff --git a/utils/bump_docker.py b/utils/bump_docker.py
new file mode 100644
index 0000000..ec4a1c7
--- /dev/null
+++ b/utils/bump_docker.py
@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+
+"""Usage: bump_docker.py [VERSION]"""
+
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+VERSION_PATTERN = r'\d+\.\d+\.\d+'
+
+if not sys.argv[1:] or re.match(VERSION_PATTERN, sys.argv[1]) is None:
+    print(__doc__)
+    raise SystemExit(1)
+
+VERSION = sys.argv[1]
+
+PROJECTS_ROOT = Path(__file__).resolve().parents[2]
+DOCKER_ROOT = PROJECTS_ROOT / 'sphinx-docker-images'
+DOCKERFILE_BASE = DOCKER_ROOT / 'base' / 'Dockerfile'
+DOCKERFILE_LATEXPDF = DOCKER_ROOT / 'latexpdf' / 'Dockerfile'
+
+OPENCONTAINERS_VERSION_PREFIX = 'LABEL org.opencontainers.image.version'
+SPHINX_VERSION_PREFIX = 'Sphinx=='
+
+for file in DOCKERFILE_BASE, DOCKERFILE_LATEXPDF:
+    content = file.read_text(encoding='utf-8')
+    content = re.sub(rf'{re.escape(OPENCONTAINERS_VERSION_PREFIX)} = "{VERSION_PATTERN}"',
+                     rf'{OPENCONTAINERS_VERSION_PREFIX} = "{VERSION}"',
+                     content)
+    content = re.sub(rf'{re.escape(SPHINX_VERSION_PREFIX)}{VERSION_PATTERN}',
+                     rf'{SPHINX_VERSION_PREFIX}{VERSION}',
+                     content)
+    file.write_text(content, encoding='utf-8')
+
+
+def git(*args):
+    ret = subprocess.run(('git', *args),
+                         capture_output=True,
+                         cwd=DOCKER_ROOT,
+                         check=True,
+                         text=True,
+                         encoding='utf-8')
+    print(ret.stdout)
+    print(ret.stderr, file=sys.stderr)
+
+
+git('checkout', 'master')
+git('commit', '-am', f'Bump to {VERSION}')
+git('tag', VERSION, '-m', f'Sphinx {VERSION}')
+git('push', 'upstream', 'master')
+git('push', 'upstream', VERSION)
diff --git a/utils/bump_version.py b/utils/bump_version.py
new file mode 100755
index 0000000..6e50755
--- /dev/null
+++ b/utils/bump_version.py
@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import re
+import sys
+import time
+from contextlib import contextmanager
+
+script_dir = os.path.dirname(__file__)
+package_dir = os.path.abspath(os.path.join(script_dir, '..'))
+
+RELEASE_TYPE = {'a': 'alpha', 'b': 'beta'}
+
+
+def stringify_version(version_info, in_develop=True):
+    version = '.'.join(str(v) for v in version_info[:3])
+    if not in_develop and version_info[3] != 'final':
+        version += version_info[3][0] + str(version_info[4])
+
+    return version
+
+
+def bump_version(path, version_info, in_develop=True):
+    version = stringify_version(version_info, in_develop)
+
+    with open(path, encoding='utf-8') as f:
+        lines = f.read().splitlines()
+
+    for i, line in enumerate(lines):
+        if line.startswith('__version__ = '):
+            lines[i] = f"__version__ = '{version}'"
+            continue
+        if line.startswith('version_info = '):
+            lines[i] = f'version_info = {version_info}'
+            continue
+        if line.startswith('_in_development = '):
+            lines[i] = f'_in_development = {in_develop}'
+            continue
+
+    with open(path, 'w', encoding='utf-8') as f:
+        f.write('\n'.join(lines) + '\n')
+
+
+def parse_version(version):
+    matched = re.search(r'^(\d+)\.(\d+)$', version)
+    if matched:
+        major, minor = matched.groups()
+        return (int(major), int(minor), 0, 'final', 0)
+
+    matched = re.search(r'^(\d+)\.(\d+)\.(\d+)$', version)
+    if matched:
+        major, minor, rev = matched.groups()
+        return (int(major), int(minor), int(rev), 'final', 0)
+
+    matched = re.search(r'^(\d+)\.(\d+)\s*(a|b|alpha|beta)(\d+)$', version)
+    if matched:
+        major, minor, typ, relver = matched.groups()
+        release = RELEASE_TYPE.get(typ, typ)
+        return (int(major), int(minor), 0, release, int(relver))
+
+    matched = re.search(r'^(\d+)\.(\d+)\.(\d+)\s*(a|b|alpha|beta)(\d+)$', version)
+    if matched:
+        major, minor, rev, typ, relver = matched.groups()
+        release = RELEASE_TYPE.get(typ, typ)
+        return (int(major), int(minor), int(rev), release, int(relver))
+
+    raise RuntimeError('Unknown version: %s' % version)
+
+
+class Skip(Exception):
+    pass
+
+
+@contextmanager
+def processing(message):
+    try:
+        print(message + ' ... ', end='')
+        yield
+    except Skip as exc:
+        print('skip: %s' % exc)
+    except Exception:
+        print('error')
+        raise
+    else:
+        print('done')
+
+
+class Changes:
+    def __init__(self, path):
+        self.path = path
+        self.fetch_version()
+
+    def fetch_version(self):
+        with open(self.path, encoding='utf-8') as f:
+            version = f.readline().strip()
+            matched = re.search(r'^Release (.*) \((.*)\)$', version)
+            if matched is None:
+                raise RuntimeError('Unknown CHANGES format: %s' % version)
+
+            self.version, self.release_date = matched.groups()
+            self.version_info = parse_version(self.version)
+            if self.release_date == 'in development':
+                self.in_development = True
+            else:
+                self.in_development = False
+
+    def finalize_release_date(self):
+        release_date = time.strftime('%b %d, %Y')
+        heading = f'Release {self.version} (released {release_date})'
+
+        with open(self.path, 'r+', encoding='utf-8') as f:
+            f.readline()  # skip first two lines
+            f.readline()
+            body = f.read()
+
+            f.seek(0)
+            f.truncate(0)
+            f.write(heading + '\n')
+            f.write('=' * len(heading) + '\n')
+            f.write(self.filter_empty_sections(body))
+
+    def add_release(self, version_info):
+        if version_info[-2:] in (('beta', 0), ('final', 0)):
+            version = stringify_version(version_info)
+        else:
+            reltype = version_info[3]
+            version = (f'{stringify_version(version_info)} '
+                       f'{RELEASE_TYPE.get(reltype, reltype)}{version_info[4] or ""}')
+        heading = 'Release %s (in development)' % version
+
+        with open(os.path.join(script_dir, 'CHANGES_template'), encoding='utf-8') as f:
+            f.readline()  # skip first two lines
+            f.readline()
+            tmpl = f.read()
+
+        with open(self.path, 'r+', encoding='utf-8') as f:
+            body = f.read()
+
+            f.seek(0)
+            f.truncate(0)
+            f.write(heading + '\n')
+            f.write('=' * len(heading) + '\n')
+            f.write(tmpl)
+            f.write('\n')
+            f.write(body)
+
+    def filter_empty_sections(self, body):
+        return re.sub('^\n.+\n-{3,}\n+(?=\n.+\n[-=]{3,}\n)', '', body, flags=re.M)
+
+
+def parse_options(argv):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('version', help='A version number (cf. 1.6b0)')
+    parser.add_argument('--in-develop', action='store_true')
+    options = parser.parse_args(argv)
+    options.version = parse_version(options.version)
+    return options
+
+
+def main():
+    options = parse_options(sys.argv[1:])
+
+    with processing("Rewriting sphinx/__init__.py"):
+        bump_version(os.path.join(package_dir, 'sphinx/__init__.py'),
+                     options.version, options.in_develop)
+
+    with processing('Rewriting CHANGES'):
+        changes = Changes(os.path.join(package_dir, 'CHANGES'))
+        if changes.version_info == options.version:
+            if changes.in_development:
+                changes.finalize_release_date()
+            else:
+                reason = 'version not changed'
+                raise Skip(reason)
+        else:
+            if changes.in_development:
+                print('WARNING: last version is not released yet: %s' % changes.version)
+            changes.add_release(options.version)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/utils/release-checklist b/utils/release-checklist
new file mode 100644
index 0000000..5aabbce
--- /dev/null
+++ b/utils/release-checklist
@@ -0,0 +1,70 @@
+Release checklist
+=================
+
+A stable release is a release where the minor or micro version parts are
+incremented.
+A major release is a release where the major version part is incremented.
+
+Checks
+------
+
+* open "https://github.com/sphinx-doc/sphinx/actions?query=branch:master" and all tests has passed
+* Run ``git fetch; git status`` and check that nothing has changed
+
+Bump version
+------------
+
+for stable and major releases
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* ``python utils/bump_version.py X.Y.Z``
+* Check diff by ``git diff``
+* ``git commit -am 'Bump to X.Y.Z final'``
+* ``git tag vX.Y.Z -m "Sphinx X.Y.Z"``
+
+for beta releases
+~~~~~~~~~~~~~~~~~
+
+* ``python utils/bump_version.py X.Y.0bN``
+* Check diff by ``git diff``
+* ``git commit -am 'Bump to X.Y.0 betaN'``
+* ``git tag vX.Y.0b1 -m "Sphinx X.Y.0bN"``
+
+Build Sphinx
+------------
+
+* ``make clean``
+* ``python -m build .``
+* ``twine upload dist/Sphinx-*``
+* open https://pypi.org/project/Sphinx/ and check for any obvious errors
+
+for stable and major releases
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* ``sh utils/bump_docker.sh X.Y.Z``
+
+Bump to next development version
+--------------------------------
+
+for stable and major releases
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* ``python utils/bump_version.py --in-develop X.Y.Z+1b0`` (ex. 1.5.3b0)
+
+for beta releases
+~~~~~~~~~~~~~~~~~
+
+* ``python utils/bump_version.py --in-develop X.Y.0bN+1`` (ex. 1.6.0b2)
+
+Commit version bump
+-------------------
+
+* Check diff by ``git diff``
+* ``git commit -am 'Bump version'``
+* ``git push origin master --tags``
+
+Final steps
+-----------
+
+* Add new version/milestone to tracker categories
+* Write announcement and send to sphinx-dev, sphinx-users and python-announce