diff options
Diffstat (limited to '')
| -rw-r--r-- | utils/CHANGES_template | 20 | ||||
| -rw-r--r-- | utils/__init__.py | 0 | ||||
| -rw-r--r-- | utils/babel_runner.py | 250 | ||||
| -rw-r--r-- | utils/bump_docker.py | 52 | ||||
| -rwxr-xr-x | utils/bump_version.py | 183 | ||||
| -rw-r--r-- | utils/release-checklist | 70 |
6 files changed, 575 insertions, 0 deletions
diff --git a/utils/CHANGES_template b/utils/CHANGES_template new file mode 100644 index 0000000..a655c46 --- /dev/null +++ b/utils/CHANGES_template @@ -0,0 +1,20 @@ +Release x.y.z (in development) +============================== + +Dependencies +------------ + +Incompatible changes +-------------------- + +Deprecated +---------- + +Features added +-------------- + +Bugs fixed +---------- + +Testing +------- diff --git a/utils/__init__.py b/utils/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/utils/__init__.py diff --git a/utils/babel_runner.py b/utils/babel_runner.py new file mode 100644 index 0000000..dfb58db --- /dev/null +++ b/utils/babel_runner.py @@ -0,0 +1,250 @@ +"""Run babel for translations. + +Usage: + +babel_runner.py extract + Extract messages from the source code and update the ".pot" template file. + +babel_runner.py update + Update all language catalogues in "sphinx/locale/<language>/LC_MESSAGES" + with the current messages in the template file. + +babel_runner.py compile + Compile the ".po" catalogue files to ".mo" and ".js" files. +""" + +import json +import logging +import os +import sys +import tempfile + +from babel.messages.catalog import Catalog +from babel.messages.extract import ( + DEFAULT_KEYWORDS, + extract, + extract_javascript, + extract_python, +) +from babel.messages.mofile import write_mo +from babel.messages.pofile import read_po, write_po +from babel.util import pathmatch +from jinja2.ext import babel_extract as extract_jinja2 + +ROOT = os.path.realpath(os.path.join(os.path.abspath(__file__), "..", "..")) +TEX_DELIMITERS = { + 'variable_start_string': '<%=', + 'variable_end_string': '%>', + 'block_start_string': '<%', + 'block_end_string': '%>', +} +METHOD_MAP = [ + # Extraction from Python source files + ('**.py', extract_python), + # Extraction from Jinja2 template files + ('**/templates/latex/**.tex_t', extract_jinja2), + ('**/templates/latex/**.sty_t', extract_jinja2), + # Extraction from Jinja2 HTML templates + ('**/themes/**.html', extract_jinja2), + # Extraction from Jinja2 XML templates + ('**/themes/**.xml', extract_jinja2), + # Extraction from JavaScript files + ('**.js', extract_javascript), + ('**.js_t', extract_javascript), +] +OPTIONS_MAP = { + # Extraction from Python source files + '**.py': { + 'encoding': 'utf-8', + }, + # Extraction from Jinja2 template files + '**/templates/latex/**.tex_t': TEX_DELIMITERS.copy(), + '**/templates/latex/**.sty_t': TEX_DELIMITERS.copy(), + # Extraction from Jinja2 HTML templates + '**/themes/**.html': { + 'encoding': 'utf-8', + 'ignore_tags': 'script,style', + 'include_attrs': 'alt title summary', + }, +} +KEYWORDS = {**DEFAULT_KEYWORDS, '_': None, '__': None} + + +def run_extract(): + """Message extraction function.""" + log = _get_logger() + + with open('sphinx/__init__.py', encoding='utf-8') as f: + for line in f.read().splitlines(): + if line.startswith('__version__ = '): + # remove prefix; strip whitespace; remove quotation marks + sphinx_version = line[14:].strip()[1:-1] + break + + input_path = 'sphinx' + catalogue = Catalog(project='Sphinx', version=sphinx_version, charset='utf-8') + + base = os.path.abspath(input_path) + for root, dirnames, filenames in os.walk(base): + relative_root = os.path.relpath(root, base) if root != base else '' + dirnames.sort() + for filename in sorted(filenames): + relative_name = os.path.join(relative_root, filename) + for pattern, method in METHOD_MAP: + if not pathmatch(pattern, relative_name): + continue + + options = {} + for opt_pattern, opt_dict in OPTIONS_MAP.items(): + if pathmatch(opt_pattern, relative_name): + options = opt_dict + with open(os.path.join(root, filename), 'rb') as fileobj: + for lineno, message, comments, context in extract( + method, fileobj, KEYWORDS, options=options, + ): + filepath = os.path.join(input_path, relative_name) + catalogue.add( + message, None, [(filepath, lineno)], + auto_comments=comments, context=context, + ) + break + + output_file = os.path.join('sphinx', 'locale', 'sphinx.pot') + log.info('writing PO template file to %s', output_file) + with open(output_file, 'wb') as outfile: + write_po(outfile, catalogue) + + +def run_update(): + """Catalog merging command.""" + + log = _get_logger() + + domain = 'sphinx' + locale_dir = os.path.join('sphinx', 'locale') + template_file = os.path.join(locale_dir, 'sphinx.pot') + + with open(template_file, encoding='utf-8') as infile: + template = read_po(infile) + + for locale in os.listdir(locale_dir): + filename = os.path.join(locale_dir, locale, 'LC_MESSAGES', f'{domain}.po') + if not os.path.exists(filename): + continue + + log.info('updating catalog %s based on %s', filename, template_file) + with open(filename, encoding='utf-8') as infile: + catalog = read_po(infile, locale=locale, domain=domain) + + catalog.update(template) + tmp_name = os.path.join( + os.path.dirname(filename), tempfile.gettempprefix() + os.path.basename(filename), + ) + try: + with open(tmp_name, 'wb') as tmpfile: + write_po(tmpfile, catalog) + except Exception: + os.remove(tmp_name) + raise + + os.replace(tmp_name, filename) + + +def run_compile(): + """ + Catalog compilation command. + + An extended command that writes all message strings that occur in + JavaScript files to a JavaScript file along with the .mo file. + + Unfortunately, babel's setup command isn't built very extensible, so + most of the run() code is duplicated here. + """ + + log = _get_logger() + + directory = os.path.join('sphinx', 'locale') + total_errors = 0 + + for locale in os.listdir(directory): + po_file = os.path.join(directory, locale, 'LC_MESSAGES', 'sphinx.po') + if not os.path.exists(po_file): + continue + + with open(po_file, encoding='utf-8') as infile: + catalog = read_po(infile, locale) + + if catalog.fuzzy: + log.info('catalog %s is marked as fuzzy, skipping', po_file) + continue + + for message, errors in catalog.check(): + for error in errors: + total_errors += 1 + log.error('error: %s:%d: %s\nerror: in message string: %s', + po_file, message.lineno, error, message.string) + + mo_file = os.path.join(directory, locale, 'LC_MESSAGES', 'sphinx.mo') + log.info('compiling catalog %s to %s', po_file, mo_file) + with open(mo_file, 'wb') as outfile: + write_mo(outfile, catalog, use_fuzzy=False) + + js_file = os.path.join(directory, locale, 'LC_MESSAGES', 'sphinx.js') + log.info('writing JavaScript strings in catalog %s to %s', po_file, js_file) + js_catalogue = {} + for message in catalog: + if any( + x[0].endswith(('.js', '.js.jinja', '.js_t', '.html')) + for x in message.locations + ): + msgid = message.id + if isinstance(msgid, (list, tuple)): + msgid = msgid[0] + js_catalogue[msgid] = message.string + + obj = json.dumps({ + 'messages': js_catalogue, + 'plural_expr': catalog.plural_expr, + 'locale': str(catalog.locale), + }, sort_keys=True, indent=4) + with open(js_file, 'wb') as outfile: + # to ensure lines end with ``\n`` rather than ``\r\n``: + outfile.write(f'Documentation.addTranslations({obj});'.encode()) + + if total_errors > 0: + log.error('%d errors encountered.', total_errors) + print("Compiling failed.", file=sys.stderr) + raise SystemExit(2) + + +def _get_logger(): + log = logging.getLogger('babel') + log.setLevel(logging.INFO) + handler = logging.StreamHandler() + handler.setFormatter(logging.Formatter('%(message)s')) + log.addHandler(handler) + return log + + +if __name__ == '__main__': + try: + action = sys.argv[1].lower() + except IndexError: + print(__doc__, file=sys.stderr) + raise SystemExit(2) from None + + os.chdir(ROOT) + if action == "extract": + raise SystemExit(run_extract()) + if action == "update": + raise SystemExit(run_update()) + if action == "compile": + raise SystemExit(run_compile()) + if action == "all": + exit_code = run_extract() + if exit_code: + raise SystemExit(exit_code) + exit_code = run_update() + if exit_code: + raise SystemExit(exit_code) + raise SystemExit(run_compile()) diff --git a/utils/bump_docker.py b/utils/bump_docker.py new file mode 100644 index 0000000..ec4a1c7 --- /dev/null +++ b/utils/bump_docker.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 + +"""Usage: bump_docker.py [VERSION]""" + +import re +import subprocess +import sys +from pathlib import Path + +VERSION_PATTERN = r'\d+\.\d+\.\d+' + +if not sys.argv[1:] or re.match(VERSION_PATTERN, sys.argv[1]) is None: + print(__doc__) + raise SystemExit(1) + +VERSION = sys.argv[1] + +PROJECTS_ROOT = Path(__file__).resolve().parents[2] +DOCKER_ROOT = PROJECTS_ROOT / 'sphinx-docker-images' +DOCKERFILE_BASE = DOCKER_ROOT / 'base' / 'Dockerfile' +DOCKERFILE_LATEXPDF = DOCKER_ROOT / 'latexpdf' / 'Dockerfile' + +OPENCONTAINERS_VERSION_PREFIX = 'LABEL org.opencontainers.image.version' +SPHINX_VERSION_PREFIX = 'Sphinx==' + +for file in DOCKERFILE_BASE, DOCKERFILE_LATEXPDF: + content = file.read_text(encoding='utf-8') + content = re.sub(rf'{re.escape(OPENCONTAINERS_VERSION_PREFIX)} = "{VERSION_PATTERN}"', + rf'{OPENCONTAINERS_VERSION_PREFIX} = "{VERSION}"', + content) + content = re.sub(rf'{re.escape(SPHINX_VERSION_PREFIX)}{VERSION_PATTERN}', + rf'{SPHINX_VERSION_PREFIX}{VERSION}', + content) + file.write_text(content, encoding='utf-8') + + +def git(*args): + ret = subprocess.run(('git', *args), + capture_output=True, + cwd=DOCKER_ROOT, + check=True, + text=True, + encoding='utf-8') + print(ret.stdout) + print(ret.stderr, file=sys.stderr) + + +git('checkout', 'master') +git('commit', '-am', f'Bump to {VERSION}') +git('tag', VERSION, '-m', f'Sphinx {VERSION}') +git('push', 'upstream', 'master') +git('push', 'upstream', VERSION) diff --git a/utils/bump_version.py b/utils/bump_version.py new file mode 100755 index 0000000..6e50755 --- /dev/null +++ b/utils/bump_version.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 + +import argparse +import os +import re +import sys +import time +from contextlib import contextmanager + +script_dir = os.path.dirname(__file__) +package_dir = os.path.abspath(os.path.join(script_dir, '..')) + +RELEASE_TYPE = {'a': 'alpha', 'b': 'beta'} + + +def stringify_version(version_info, in_develop=True): + version = '.'.join(str(v) for v in version_info[:3]) + if not in_develop and version_info[3] != 'final': + version += version_info[3][0] + str(version_info[4]) + + return version + + +def bump_version(path, version_info, in_develop=True): + version = stringify_version(version_info, in_develop) + + with open(path, encoding='utf-8') as f: + lines = f.read().splitlines() + + for i, line in enumerate(lines): + if line.startswith('__version__ = '): + lines[i] = f"__version__ = '{version}'" + continue + if line.startswith('version_info = '): + lines[i] = f'version_info = {version_info}' + continue + if line.startswith('_in_development = '): + lines[i] = f'_in_development = {in_develop}' + continue + + with open(path, 'w', encoding='utf-8') as f: + f.write('\n'.join(lines) + '\n') + + +def parse_version(version): + matched = re.search(r'^(\d+)\.(\d+)$', version) + if matched: + major, minor = matched.groups() + return (int(major), int(minor), 0, 'final', 0) + + matched = re.search(r'^(\d+)\.(\d+)\.(\d+)$', version) + if matched: + major, minor, rev = matched.groups() + return (int(major), int(minor), int(rev), 'final', 0) + + matched = re.search(r'^(\d+)\.(\d+)\s*(a|b|alpha|beta)(\d+)$', version) + if matched: + major, minor, typ, relver = matched.groups() + release = RELEASE_TYPE.get(typ, typ) + return (int(major), int(minor), 0, release, int(relver)) + + matched = re.search(r'^(\d+)\.(\d+)\.(\d+)\s*(a|b|alpha|beta)(\d+)$', version) + if matched: + major, minor, rev, typ, relver = matched.groups() + release = RELEASE_TYPE.get(typ, typ) + return (int(major), int(minor), int(rev), release, int(relver)) + + raise RuntimeError('Unknown version: %s' % version) + + +class Skip(Exception): + pass + + +@contextmanager +def processing(message): + try: + print(message + ' ... ', end='') + yield + except Skip as exc: + print('skip: %s' % exc) + except Exception: + print('error') + raise + else: + print('done') + + +class Changes: + def __init__(self, path): + self.path = path + self.fetch_version() + + def fetch_version(self): + with open(self.path, encoding='utf-8') as f: + version = f.readline().strip() + matched = re.search(r'^Release (.*) \((.*)\)$', version) + if matched is None: + raise RuntimeError('Unknown CHANGES format: %s' % version) + + self.version, self.release_date = matched.groups() + self.version_info = parse_version(self.version) + if self.release_date == 'in development': + self.in_development = True + else: + self.in_development = False + + def finalize_release_date(self): + release_date = time.strftime('%b %d, %Y') + heading = f'Release {self.version} (released {release_date})' + + with open(self.path, 'r+', encoding='utf-8') as f: + f.readline() # skip first two lines + f.readline() + body = f.read() + + f.seek(0) + f.truncate(0) + f.write(heading + '\n') + f.write('=' * len(heading) + '\n') + f.write(self.filter_empty_sections(body)) + + def add_release(self, version_info): + if version_info[-2:] in (('beta', 0), ('final', 0)): + version = stringify_version(version_info) + else: + reltype = version_info[3] + version = (f'{stringify_version(version_info)} ' + f'{RELEASE_TYPE.get(reltype, reltype)}{version_info[4] or ""}') + heading = 'Release %s (in development)' % version + + with open(os.path.join(script_dir, 'CHANGES_template'), encoding='utf-8') as f: + f.readline() # skip first two lines + f.readline() + tmpl = f.read() + + with open(self.path, 'r+', encoding='utf-8') as f: + body = f.read() + + f.seek(0) + f.truncate(0) + f.write(heading + '\n') + f.write('=' * len(heading) + '\n') + f.write(tmpl) + f.write('\n') + f.write(body) + + def filter_empty_sections(self, body): + return re.sub('^\n.+\n-{3,}\n+(?=\n.+\n[-=]{3,}\n)', '', body, flags=re.M) + + +def parse_options(argv): + parser = argparse.ArgumentParser() + parser.add_argument('version', help='A version number (cf. 1.6b0)') + parser.add_argument('--in-develop', action='store_true') + options = parser.parse_args(argv) + options.version = parse_version(options.version) + return options + + +def main(): + options = parse_options(sys.argv[1:]) + + with processing("Rewriting sphinx/__init__.py"): + bump_version(os.path.join(package_dir, 'sphinx/__init__.py'), + options.version, options.in_develop) + + with processing('Rewriting CHANGES'): + changes = Changes(os.path.join(package_dir, 'CHANGES')) + if changes.version_info == options.version: + if changes.in_development: + changes.finalize_release_date() + else: + reason = 'version not changed' + raise Skip(reason) + else: + if changes.in_development: + print('WARNING: last version is not released yet: %s' % changes.version) + changes.add_release(options.version) + + +if __name__ == '__main__': + main() diff --git a/utils/release-checklist b/utils/release-checklist new file mode 100644 index 0000000..5aabbce --- /dev/null +++ b/utils/release-checklist @@ -0,0 +1,70 @@ +Release checklist +================= + +A stable release is a release where the minor or micro version parts are +incremented. +A major release is a release where the major version part is incremented. + +Checks +------ + +* open "https://github.com/sphinx-doc/sphinx/actions?query=branch:master" and all tests has passed +* Run ``git fetch; git status`` and check that nothing has changed + +Bump version +------------ + +for stable and major releases +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* ``python utils/bump_version.py X.Y.Z`` +* Check diff by ``git diff`` +* ``git commit -am 'Bump to X.Y.Z final'`` +* ``git tag vX.Y.Z -m "Sphinx X.Y.Z"`` + +for beta releases +~~~~~~~~~~~~~~~~~ + +* ``python utils/bump_version.py X.Y.0bN`` +* Check diff by ``git diff`` +* ``git commit -am 'Bump to X.Y.0 betaN'`` +* ``git tag vX.Y.0b1 -m "Sphinx X.Y.0bN"`` + +Build Sphinx +------------ + +* ``make clean`` +* ``python -m build .`` +* ``twine upload dist/Sphinx-*`` +* open https://pypi.org/project/Sphinx/ and check for any obvious errors + +for stable and major releases +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* ``sh utils/bump_docker.sh X.Y.Z`` + +Bump to next development version +-------------------------------- + +for stable and major releases +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* ``python utils/bump_version.py --in-develop X.Y.Z+1b0`` (ex. 1.5.3b0) + +for beta releases +~~~~~~~~~~~~~~~~~ + +* ``python utils/bump_version.py --in-develop X.Y.0bN+1`` (ex. 1.6.0b2) + +Commit version bump +------------------- + +* Check diff by ``git diff`` +* ``git commit -am 'Bump version'`` +* ``git push origin master --tags`` + +Final steps +----------- + +* Add new version/milestone to tracker categories +* Write announcement and send to sphinx-dev, sphinx-users and python-announce |