summaryrefslogtreecommitdiffstats
path: root/utils/babel_runner.py
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-15 17:25:40 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-15 17:25:40 +0000
commitcf7da1843c45a4c2df7a749f7886a2d2ba0ee92a (patch)
tree18dcde1a8d1f5570a77cd0c361de3b490d02c789 /utils/babel_runner.py
parentInitial commit. (diff)
downloadsphinx-cf7da1843c45a4c2df7a749f7886a2d2ba0ee92a.tar.xz
sphinx-cf7da1843c45a4c2df7a749f7886a2d2ba0ee92a.zip
Adding upstream version 7.2.6.upstream/7.2.6
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'utils/babel_runner.py')
-rw-r--r--utils/babel_runner.py250
1 files changed, 250 insertions, 0 deletions
diff --git a/utils/babel_runner.py b/utils/babel_runner.py
new file mode 100644
index 0000000..dfb58db
--- /dev/null
+++ b/utils/babel_runner.py
@@ -0,0 +1,250 @@
+"""Run babel for translations.
+
+Usage:
+
+babel_runner.py extract
+ Extract messages from the source code and update the ".pot" template file.
+
+babel_runner.py update
+ Update all language catalogues in "sphinx/locale/<language>/LC_MESSAGES"
+ with the current messages in the template file.
+
+babel_runner.py compile
+ Compile the ".po" catalogue files to ".mo" and ".js" files.
+"""
+
+import json
+import logging
+import os
+import sys
+import tempfile
+
+from babel.messages.catalog import Catalog
+from babel.messages.extract import (
+ DEFAULT_KEYWORDS,
+ extract,
+ extract_javascript,
+ extract_python,
+)
+from babel.messages.mofile import write_mo
+from babel.messages.pofile import read_po, write_po
+from babel.util import pathmatch
+from jinja2.ext import babel_extract as extract_jinja2
+
+ROOT = os.path.realpath(os.path.join(os.path.abspath(__file__), "..", ".."))
+TEX_DELIMITERS = {
+ 'variable_start_string': '<%=',
+ 'variable_end_string': '%>',
+ 'block_start_string': '<%',
+ 'block_end_string': '%>',
+}
+METHOD_MAP = [
+ # Extraction from Python source files
+ ('**.py', extract_python),
+ # Extraction from Jinja2 template files
+ ('**/templates/latex/**.tex_t', extract_jinja2),
+ ('**/templates/latex/**.sty_t', extract_jinja2),
+ # Extraction from Jinja2 HTML templates
+ ('**/themes/**.html', extract_jinja2),
+ # Extraction from Jinja2 XML templates
+ ('**/themes/**.xml', extract_jinja2),
+ # Extraction from JavaScript files
+ ('**.js', extract_javascript),
+ ('**.js_t', extract_javascript),
+]
+OPTIONS_MAP = {
+ # Extraction from Python source files
+ '**.py': {
+ 'encoding': 'utf-8',
+ },
+ # Extraction from Jinja2 template files
+ '**/templates/latex/**.tex_t': TEX_DELIMITERS.copy(),
+ '**/templates/latex/**.sty_t': TEX_DELIMITERS.copy(),
+ # Extraction from Jinja2 HTML templates
+ '**/themes/**.html': {
+ 'encoding': 'utf-8',
+ 'ignore_tags': 'script,style',
+ 'include_attrs': 'alt title summary',
+ },
+}
+KEYWORDS = {**DEFAULT_KEYWORDS, '_': None, '__': None}
+
+
+def run_extract():
+ """Message extraction function."""
+ log = _get_logger()
+
+ with open('sphinx/__init__.py', encoding='utf-8') as f:
+ for line in f.read().splitlines():
+ if line.startswith('__version__ = '):
+ # remove prefix; strip whitespace; remove quotation marks
+ sphinx_version = line[14:].strip()[1:-1]
+ break
+
+ input_path = 'sphinx'
+ catalogue = Catalog(project='Sphinx', version=sphinx_version, charset='utf-8')
+
+ base = os.path.abspath(input_path)
+ for root, dirnames, filenames in os.walk(base):
+ relative_root = os.path.relpath(root, base) if root != base else ''
+ dirnames.sort()
+ for filename in sorted(filenames):
+ relative_name = os.path.join(relative_root, filename)
+ for pattern, method in METHOD_MAP:
+ if not pathmatch(pattern, relative_name):
+ continue
+
+ options = {}
+ for opt_pattern, opt_dict in OPTIONS_MAP.items():
+ if pathmatch(opt_pattern, relative_name):
+ options = opt_dict
+ with open(os.path.join(root, filename), 'rb') as fileobj:
+ for lineno, message, comments, context in extract(
+ method, fileobj, KEYWORDS, options=options,
+ ):
+ filepath = os.path.join(input_path, relative_name)
+ catalogue.add(
+ message, None, [(filepath, lineno)],
+ auto_comments=comments, context=context,
+ )
+ break
+
+ output_file = os.path.join('sphinx', 'locale', 'sphinx.pot')
+ log.info('writing PO template file to %s', output_file)
+ with open(output_file, 'wb') as outfile:
+ write_po(outfile, catalogue)
+
+
+def run_update():
+ """Catalog merging command."""
+
+ log = _get_logger()
+
+ domain = 'sphinx'
+ locale_dir = os.path.join('sphinx', 'locale')
+ template_file = os.path.join(locale_dir, 'sphinx.pot')
+
+ with open(template_file, encoding='utf-8') as infile:
+ template = read_po(infile)
+
+ for locale in os.listdir(locale_dir):
+ filename = os.path.join(locale_dir, locale, 'LC_MESSAGES', f'{domain}.po')
+ if not os.path.exists(filename):
+ continue
+
+ log.info('updating catalog %s based on %s', filename, template_file)
+ with open(filename, encoding='utf-8') as infile:
+ catalog = read_po(infile, locale=locale, domain=domain)
+
+ catalog.update(template)
+ tmp_name = os.path.join(
+ os.path.dirname(filename), tempfile.gettempprefix() + os.path.basename(filename),
+ )
+ try:
+ with open(tmp_name, 'wb') as tmpfile:
+ write_po(tmpfile, catalog)
+ except Exception:
+ os.remove(tmp_name)
+ raise
+
+ os.replace(tmp_name, filename)
+
+
+def run_compile():
+ """
+ Catalog compilation command.
+
+ An extended command that writes all message strings that occur in
+ JavaScript files to a JavaScript file along with the .mo file.
+
+ Unfortunately, babel's setup command isn't built very extensible, so
+ most of the run() code is duplicated here.
+ """
+
+ log = _get_logger()
+
+ directory = os.path.join('sphinx', 'locale')
+ total_errors = 0
+
+ for locale in os.listdir(directory):
+ po_file = os.path.join(directory, locale, 'LC_MESSAGES', 'sphinx.po')
+ if not os.path.exists(po_file):
+ continue
+
+ with open(po_file, encoding='utf-8') as infile:
+ catalog = read_po(infile, locale)
+
+ if catalog.fuzzy:
+ log.info('catalog %s is marked as fuzzy, skipping', po_file)
+ continue
+
+ for message, errors in catalog.check():
+ for error in errors:
+ total_errors += 1
+ log.error('error: %s:%d: %s\nerror: in message string: %s',
+ po_file, message.lineno, error, message.string)
+
+ mo_file = os.path.join(directory, locale, 'LC_MESSAGES', 'sphinx.mo')
+ log.info('compiling catalog %s to %s', po_file, mo_file)
+ with open(mo_file, 'wb') as outfile:
+ write_mo(outfile, catalog, use_fuzzy=False)
+
+ js_file = os.path.join(directory, locale, 'LC_MESSAGES', 'sphinx.js')
+ log.info('writing JavaScript strings in catalog %s to %s', po_file, js_file)
+ js_catalogue = {}
+ for message in catalog:
+ if any(
+ x[0].endswith(('.js', '.js.jinja', '.js_t', '.html'))
+ for x in message.locations
+ ):
+ msgid = message.id
+ if isinstance(msgid, (list, tuple)):
+ msgid = msgid[0]
+ js_catalogue[msgid] = message.string
+
+ obj = json.dumps({
+ 'messages': js_catalogue,
+ 'plural_expr': catalog.plural_expr,
+ 'locale': str(catalog.locale),
+ }, sort_keys=True, indent=4)
+ with open(js_file, 'wb') as outfile:
+ # to ensure lines end with ``\n`` rather than ``\r\n``:
+ outfile.write(f'Documentation.addTranslations({obj});'.encode())
+
+ if total_errors > 0:
+ log.error('%d errors encountered.', total_errors)
+ print("Compiling failed.", file=sys.stderr)
+ raise SystemExit(2)
+
+
+def _get_logger():
+ log = logging.getLogger('babel')
+ log.setLevel(logging.INFO)
+ handler = logging.StreamHandler()
+ handler.setFormatter(logging.Formatter('%(message)s'))
+ log.addHandler(handler)
+ return log
+
+
+if __name__ == '__main__':
+ try:
+ action = sys.argv[1].lower()
+ except IndexError:
+ print(__doc__, file=sys.stderr)
+ raise SystemExit(2) from None
+
+ os.chdir(ROOT)
+ if action == "extract":
+ raise SystemExit(run_extract())
+ if action == "update":
+ raise SystemExit(run_update())
+ if action == "compile":
+ raise SystemExit(run_compile())
+ if action == "all":
+ exit_code = run_extract()
+ if exit_code:
+ raise SystemExit(exit_code)
+ exit_code = run_update()
+ if exit_code:
+ raise SystemExit(exit_code)
+ raise SystemExit(run_compile())