diff options
Diffstat (limited to 'utils')
-rw-r--r-- | utils/CHANGES_template.rst | 20 | ||||
-rw-r--r-- | utils/__init__.py | 0 | ||||
-rw-r--r-- | utils/babel_runner.py | 270 | ||||
-rwxr-xr-x | utils/bump_docker.py | 58 | ||||
-rwxr-xr-x | utils/bump_version.py | 196 | ||||
-rw-r--r-- | utils/release-checklist.rst | 70 |
6 files changed, 614 insertions, 0 deletions
diff --git a/utils/CHANGES_template.rst b/utils/CHANGES_template.rst new file mode 100644 index 0000000..a655c46 --- /dev/null +++ b/utils/CHANGES_template.rst @@ -0,0 +1,20 @@ +Release x.y.z (in development) +============================== + +Dependencies +------------ + +Incompatible changes +-------------------- + +Deprecated +---------- + +Features added +-------------- + +Bugs fixed +---------- + +Testing +------- diff --git a/utils/__init__.py b/utils/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/utils/__init__.py diff --git a/utils/babel_runner.py b/utils/babel_runner.py new file mode 100644 index 0000000..b66425f --- /dev/null +++ b/utils/babel_runner.py @@ -0,0 +1,270 @@ +"""Run babel for translations. + +Usage: + +babel_runner.py extract + Extract messages from the source code and update the ".pot" template file. + +babel_runner.py update + Update all language catalogues in "sphinx/locale/<language>/LC_MESSAGES" + with the current messages in the template file. + +babel_runner.py compile + Compile the ".po" catalogue files to ".mo" and ".js" files. +""" + +import json +import logging +import os +import sys +import tempfile + +from babel.messages.catalog import Catalog +from babel.messages.extract import ( + DEFAULT_KEYWORDS, + extract, + extract_javascript, + extract_python, +) +from babel.messages.mofile import write_mo +from babel.messages.pofile import read_po, write_po +from babel.util import pathmatch +from jinja2.ext import babel_extract as extract_jinja2 + +ROOT = os.path.realpath(os.path.join(os.path.abspath(__file__), '..', '..')) +TEX_DELIMITERS = { + 'variable_start_string': '<%=', + 'variable_end_string': '%>', + 'block_start_string': '<%', + 'block_end_string': '%>', +} +METHOD_MAP = [ + # Extraction from Python source files + ('**.py', extract_python), + # Extraction from Jinja2 template files + ('**/templates/latex/**.tex_t', extract_jinja2), + ('**/templates/latex/**.sty_t', extract_jinja2), + # Extraction from Jinja2 HTML templates + ('**/themes/**.html', extract_jinja2), + # Extraction from Jinja2 XML templates + ('**/themes/**.xml', extract_jinja2), + # Extraction from JavaScript files + ('**.js', extract_javascript), + ('**.js_t', extract_javascript), +] +OPTIONS_MAP = { + # Extraction from Python source files + '**.py': { + 'encoding': 'utf-8', + }, + # Extraction from Jinja2 template files + '**/templates/latex/**.tex_t': TEX_DELIMITERS.copy(), + '**/templates/latex/**.sty_t': TEX_DELIMITERS.copy(), + # Extraction from Jinja2 HTML templates + '**/themes/**.html': { + 'encoding': 'utf-8', + 'ignore_tags': 'script,style', + 'include_attrs': 'alt title summary', + }, +} +KEYWORDS = {**DEFAULT_KEYWORDS, '_': None, '__': None} + + +def run_extract() -> None: + """Message extraction function.""" + log = _get_logger() + + with open('sphinx/__init__.py', encoding='utf-8') as f: + for line in f.read().splitlines(): + if line.startswith('__version__ = '): + # remove prefix; strip whitespace; remove quotation marks + sphinx_version = line[14:].strip()[1:-1] + break + + input_path = 'sphinx' + catalogue = Catalog(project='Sphinx', version=sphinx_version, charset='utf-8') + + base = os.path.abspath(input_path) + for root, dirnames, filenames in os.walk(base): + relative_root = os.path.relpath(root, base) if root != base else '' + dirnames.sort() + for filename in sorted(filenames): + relative_name = os.path.join(relative_root, filename) + for pattern, method in METHOD_MAP: + if not pathmatch(pattern, relative_name): + continue + + options = {} + for opt_pattern, opt_dict in OPTIONS_MAP.items(): + if pathmatch(opt_pattern, relative_name): + options = opt_dict + with open(os.path.join(root, filename), 'rb') as fileobj: + for lineno, message, comments, context in extract( + method, fileobj, KEYWORDS, options=options + ): + filepath = os.path.join(input_path, relative_name) + catalogue.add( + message, + None, + [(filepath, lineno)], + auto_comments=comments, + context=context, + ) + break + + output_file = os.path.join('sphinx', 'locale', 'sphinx.pot') + log.info('writing PO template file to %s', output_file) + with open(output_file, 'wb') as outfile: + write_po(outfile, catalogue) + + +def run_update() -> None: + """Catalog merging command.""" + log = _get_logger() + + domain = 'sphinx' + locale_dir = os.path.join('sphinx', 'locale') + template_file = os.path.join(locale_dir, 'sphinx.pot') + + with open(template_file, encoding='utf-8') as infile: + template = read_po(infile) + + for locale in os.listdir(locale_dir): + filename = os.path.join(locale_dir, locale, 'LC_MESSAGES', f'{domain}.po') + if not os.path.exists(filename): + continue + + log.info('updating catalog %s based on %s', filename, template_file) + with open(filename, encoding='utf-8') as infile: + catalog = read_po(infile, locale=locale, domain=domain) + + catalog.update(template) + tmp_name = os.path.join( + os.path.dirname(filename), + tempfile.gettempprefix() + os.path.basename(filename), + ) + try: + with open(tmp_name, 'wb') as tmpfile: + write_po(tmpfile, catalog) + except Exception: + os.remove(tmp_name) + raise + + os.replace(tmp_name, filename) + + +def run_compile() -> None: + """ + Catalog compilation command. + + An extended command that writes all message strings that occur in + JavaScript files to a JavaScript file along with the .mo file. + + Unfortunately, babel's setup command isn't built very extensible, so + most of the run() code is duplicated here. + """ + log = _get_logger() + + directory = os.path.join('sphinx', 'locale') + total_errors = {} + + for locale in os.listdir(directory): + po_file = os.path.join(directory, locale, 'LC_MESSAGES', 'sphinx.po') + if not os.path.exists(po_file): + continue + + with open(po_file, encoding='utf-8') as infile: + catalog = read_po(infile, locale) + + if catalog.fuzzy: + log.info('catalog %s is marked as fuzzy, skipping', po_file) + continue + + for message, errors in catalog.check(): + if locale not in total_errors: + total_errors[locale] = 0 + for error in errors: + total_errors[locale] += 1 + log.error( + 'error: %s:%d: %s\nerror: in message string: %r', + po_file, + message.lineno, + error, + message.string, + ) + + mo_file = os.path.join(directory, locale, 'LC_MESSAGES', 'sphinx.mo') + log.info('compiling catalog %s to %s', po_file, mo_file) + with open(mo_file, 'wb') as outfile: + write_mo(outfile, catalog, use_fuzzy=False) + + js_file = os.path.join(directory, locale, 'LC_MESSAGES', 'sphinx.js') + log.info('writing JavaScript strings in catalog %s to %s', po_file, js_file) + js_catalogue = {} + for message in catalog: + if any( + x[0].endswith(('.js', '.js.jinja', '.js_t', '.html')) + for x in message.locations + ): + msgid = message.id + if isinstance(msgid, (list, tuple)): + msgid = msgid[0] + js_catalogue[msgid] = message.string + + obj = json.dumps( + { + 'messages': js_catalogue, + 'plural_expr': catalog.plural_expr, + 'locale': str(catalog.locale), + }, + sort_keys=True, + indent=4, + ) + with open(js_file, 'wb') as outfile: + # to ensure lines end with ``\n`` rather than ``\r\n``: + outfile.write(f'Documentation.addTranslations({obj});'.encode()) + + if 'ta' in total_errors: + # Tamil is a known failure. + err_count = total_errors.pop('ta') + log.error('%d errors encountered in %r locale.', err_count, 'ta') + + if len(total_errors) > 0: + for locale, err_count in total_errors.items(): + log.error('%d errors encountered in %r locale.', err_count, locale) + log.error('%d errors encountered.', sum(total_errors.values())) + print('Compiling failed.', file=sys.stderr) + raise SystemExit(2) + + +def _get_logger(): + log = logging.getLogger('babel') + log.setLevel(logging.INFO) + handler = logging.StreamHandler() + handler.setFormatter(logging.Formatter('%(message)s')) + log.addHandler(handler) + return log + + +if __name__ == '__main__': + try: + action = sys.argv[1].lower() + except IndexError: + print(__doc__, file=sys.stderr) + raise SystemExit(2) from None + + os.chdir(ROOT) + if action == 'extract': + run_extract() + elif action == 'update': + run_update() + elif action == 'compile': + run_compile() + elif action == 'all': + run_extract() + run_update() + run_compile() + else: + msg = f"invalid action: '{action}'" + raise ValueError(msg) + raise SystemExit diff --git a/utils/bump_docker.py b/utils/bump_docker.py new file mode 100755 index 0000000..8f385ae --- /dev/null +++ b/utils/bump_docker.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 + +"""Usage: bump_docker.py [VERSION]""" + +import re +import subprocess +import sys +from pathlib import Path + +VERSION_PATTERN = r'\d+\.\d+\.\d+' + +if not sys.argv[1:] or re.match(VERSION_PATTERN, sys.argv[1]) is None: + print(__doc__) + raise SystemExit(1) + +VERSION = sys.argv[1] + +PROJECTS_ROOT = Path(__file__).resolve().parents[2] +DOCKER_ROOT = PROJECTS_ROOT / 'sphinx-docker-images' +DOCKERFILE_BASE = DOCKER_ROOT / 'base' / 'Dockerfile' +DOCKERFILE_LATEXPDF = DOCKER_ROOT / 'latexpdf' / 'Dockerfile' + +OPENCONTAINERS_VERSION_PREFIX = 'LABEL org.opencontainers.image.version' +SPHINX_VERSION_PREFIX = 'Sphinx==' + +for file in DOCKERFILE_BASE, DOCKERFILE_LATEXPDF: + content = file.read_text(encoding='utf-8') + content = re.sub( + rf'{re.escape(OPENCONTAINERS_VERSION_PREFIX)} = "{VERSION_PATTERN}"', + rf'{OPENCONTAINERS_VERSION_PREFIX} = "{VERSION}"', + content, + ) + content = re.sub( + rf'{re.escape(SPHINX_VERSION_PREFIX)}{VERSION_PATTERN}', + rf'{SPHINX_VERSION_PREFIX}{VERSION}', + content, + ) + file.write_text(content, encoding='utf-8') + + +def git(*args: str) -> None: + ret = subprocess.run( + ('git', *args), + capture_output=True, + cwd=DOCKER_ROOT, + check=True, + text=True, + encoding='utf-8', + ) + print(ret.stdout) + print(ret.stderr, file=sys.stderr) + + +git('checkout', 'master') +git('commit', '-am', f'Bump to {VERSION}') +git('tag', VERSION, '-m', f'Sphinx {VERSION}') +git('push', 'upstream', 'master') +git('push', 'upstream', VERSION) diff --git a/utils/bump_version.py b/utils/bump_version.py new file mode 100755 index 0000000..7275cca --- /dev/null +++ b/utils/bump_version.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import re +import sys +import time +from contextlib import contextmanager +from pathlib import Path +from typing import TYPE_CHECKING + +from typing_extensions import TypeAlias + +if TYPE_CHECKING: + from collections.abc import Iterator, Sequence + +script_dir = Path(__file__).parent +package_dir = script_dir.parent + +RELEASE_TYPE = {'a': 'alpha', 'b': 'beta'} + +VersionInfo: TypeAlias = tuple[int, int, int, str, int] + + +def stringify_version(version_info: VersionInfo, in_develop: bool = True) -> str: + version = '.'.join(str(v) for v in version_info[:3]) + if not in_develop and version_info[3] != 'final': + version += version_info[3][0] + str(version_info[4]) + + return version + + +def bump_version(path: Path, version_info: VersionInfo, in_develop: bool = True) -> None: + version = stringify_version(version_info, in_develop) + + with open(path, encoding='utf-8') as f: + lines = f.read().splitlines() + + for i, line in enumerate(lines): + if line.startswith('__version__ = '): + lines[i] = f"__version__ = '{version}'" + continue + if line.startswith('version_info = '): + lines[i] = f'version_info = {version_info}' + continue + if line.startswith('_in_development = '): + lines[i] = f'_in_development = {in_develop}' + continue + + with open(path, 'w', encoding='utf-8') as f: + f.write('\n'.join(lines) + '\n') + + +def parse_version(version: str) -> VersionInfo: + matched = re.search(r'^(\d+)\.(\d+)$', version) + if matched: + major, minor = matched.groups() + return (int(major), int(minor), 0, 'final', 0) + + matched = re.search(r'^(\d+)\.(\d+)\.(\d+)$', version) + if matched: + major, minor, rev = matched.groups() + return (int(major), int(minor), int(rev), 'final', 0) + + matched = re.search(r'^(\d+)\.(\d+)\s*(a|b|alpha|beta)(\d+)$', version) + if matched: + major, minor, typ, relver = matched.groups() + release = RELEASE_TYPE.get(typ, typ) + return (int(major), int(minor), 0, release, int(relver)) + + matched = re.search(r'^(\d+)\.(\d+)\.(\d+)\s*(a|b|alpha|beta)(\d+)$', version) + if matched: + major, minor, rev, typ, relver = matched.groups() + release = RELEASE_TYPE.get(typ, typ) + return (int(major), int(minor), int(rev), release, int(relver)) + + raise RuntimeError('Unknown version: %s' % version) + + +class Skip(Exception): + pass + + +@contextmanager +def processing(message: str) -> Iterator[None]: + try: + print(message + ' ... ', end='') + yield + except Skip as exc: + print('skip: %s' % exc) + except Exception: + print('error') + raise + else: + print('done') + + +class Changes: + def __init__(self, path: Path) -> None: + self.path = path + self.fetch_version() + + def fetch_version(self) -> None: + with open(self.path, encoding='utf-8') as f: + version = f.readline().strip() + matched = re.search(r'^Release (.*) \((.*)\)$', version) + if matched is None: + raise RuntimeError('Unknown CHANGES format: %s' % version) + + self.version, self.release_date = matched.groups() + self.version_info = parse_version(self.version) + if self.release_date == 'in development': + self.in_development = True + else: + self.in_development = False + + def finalize_release_date(self) -> None: + release_date = time.strftime('%b %d, %Y') + heading = f'Release {self.version} (released {release_date})' + + with open(self.path, 'r+', encoding='utf-8') as f: + f.readline() # skip first two lines + f.readline() + body = f.read() + + f.seek(0) + f.truncate(0) + f.write(heading + '\n') + f.write('=' * len(heading) + '\n') + f.write(self.filter_empty_sections(body)) + + def add_release(self, version_info: VersionInfo) -> None: + if version_info[-2:] in (('beta', 0), ('final', 0)): + version = stringify_version(version_info) + else: + reltype = version_info[3] + version = ( + f'{stringify_version(version_info)} ' + f'{RELEASE_TYPE.get(reltype, reltype)}{version_info[4] or ""}' + ) + heading = 'Release %s (in development)' % version + + with open(script_dir / 'CHANGES_template.rst', encoding='utf-8') as f: + f.readline() # skip first two lines + f.readline() + tmpl = f.read() + + with open(self.path, 'r+', encoding='utf-8') as f: + body = f.read() + + f.seek(0) + f.truncate(0) + f.write(heading + '\n') + f.write('=' * len(heading) + '\n') + f.write(tmpl) + f.write('\n') + f.write(body) + + def filter_empty_sections(self, body: str) -> str: + return re.sub('^\n.+\n-{3,}\n+(?=\n.+\n[-=]{3,}\n)', '', body, flags=re.MULTILINE) + + +def parse_options(argv: Sequence[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument('version', help='A version number (cf. 1.6b0)') + parser.add_argument('--in-develop', action='store_true') + options = parser.parse_args(argv) + options.version = parse_version(options.version) + return options + + +def main() -> None: + options = parse_options(sys.argv[1:]) + + with processing('Rewriting sphinx/__init__.py'): + bump_version( + package_dir / 'sphinx' / '__init__.py', options.version, options.in_develop + ) + + with processing('Rewriting CHANGES'): + changes = Changes(package_dir / 'CHANGES.rst') + if changes.version_info == options.version: + if changes.in_development: + changes.finalize_release_date() + else: + reason = 'version not changed' + raise Skip(reason) + else: + if changes.in_development: + print('WARNING: last version is not released yet: %s' % changes.version) + changes.add_release(options.version) + + +if __name__ == '__main__': + main() diff --git a/utils/release-checklist.rst b/utils/release-checklist.rst new file mode 100644 index 0000000..5aabbce --- /dev/null +++ b/utils/release-checklist.rst @@ -0,0 +1,70 @@ +Release checklist +================= + +A stable release is a release where the minor or micro version parts are +incremented. +A major release is a release where the major version part is incremented. + +Checks +------ + +* open "https://github.com/sphinx-doc/sphinx/actions?query=branch:master" and all tests has passed +* Run ``git fetch; git status`` and check that nothing has changed + +Bump version +------------ + +for stable and major releases +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* ``python utils/bump_version.py X.Y.Z`` +* Check diff by ``git diff`` +* ``git commit -am 'Bump to X.Y.Z final'`` +* ``git tag vX.Y.Z -m "Sphinx X.Y.Z"`` + +for beta releases +~~~~~~~~~~~~~~~~~ + +* ``python utils/bump_version.py X.Y.0bN`` +* Check diff by ``git diff`` +* ``git commit -am 'Bump to X.Y.0 betaN'`` +* ``git tag vX.Y.0b1 -m "Sphinx X.Y.0bN"`` + +Build Sphinx +------------ + +* ``make clean`` +* ``python -m build .`` +* ``twine upload dist/Sphinx-*`` +* open https://pypi.org/project/Sphinx/ and check for any obvious errors + +for stable and major releases +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* ``sh utils/bump_docker.sh X.Y.Z`` + +Bump to next development version +-------------------------------- + +for stable and major releases +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* ``python utils/bump_version.py --in-develop X.Y.Z+1b0`` (ex. 1.5.3b0) + +for beta releases +~~~~~~~~~~~~~~~~~ + +* ``python utils/bump_version.py --in-develop X.Y.0bN+1`` (ex. 1.6.0b2) + +Commit version bump +------------------- + +* Check diff by ``git diff`` +* ``git commit -am 'Bump version'`` +* ``git push origin master --tags`` + +Final steps +----------- + +* Add new version/milestone to tracker categories +* Write announcement and send to sphinx-dev, sphinx-users and python-announce |