diff options
Diffstat (limited to '')
| -rw-r--r-- | tools/moztreedocs/mach_commands.py | 533 |
1 files changed, 533 insertions, 0 deletions
diff --git a/tools/moztreedocs/mach_commands.py b/tools/moztreedocs/mach_commands.py new file mode 100644 index 0000000000..6f44c65c20 --- /dev/null +++ b/tools/moztreedocs/mach_commands.py @@ -0,0 +1,533 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, # You can obtain one at http://mozilla.org/MPL/2.0/. + +import fnmatch +import json +import multiprocessing +import os +import re +import subprocess +import sys +import tempfile +import time +import uuid +from functools import partial +from pprint import pprint + +import mozpack.path as mozpath +import sentry_sdk +import yaml +from mach.decorators import Command, CommandArgument, SubCommand +from mach.registrar import Registrar +from mozbuild.util import memoize + +here = os.path.abspath(os.path.dirname(__file__)) +topsrcdir = os.path.abspath(os.path.dirname(os.path.dirname(here))) +DOC_ROOT = os.path.join(topsrcdir, "docs") +BASE_LINK = "http://gecko-docs.mozilla.org-l1.s3-website.us-west-2.amazonaws.com/" + + +# Helps manage in-tree documentation. + + +@Command( + "doc", + category="devenv", + virtualenv_name="docs", + description="Generate and serve documentation from the tree.", +) +@CommandArgument( + "path", + default=None, + metavar="DIRECTORY", + nargs="?", + help="Path to documentation to build and display.", +) +@CommandArgument( + "--format", default="html", dest="fmt", help="Documentation format to write." +) +@CommandArgument( + "--outdir", default=None, metavar="DESTINATION", help="Where to write output." +) +@CommandArgument( + "--archive", + action="store_true", + help="Write a gzipped tarball of generated docs.", +) +@CommandArgument( + "--no-open", + dest="auto_open", + default=True, + action="store_false", + help="Don't automatically open HTML docs in a browser.", +) +@CommandArgument( + "--no-serve", + dest="serve", + default=True, + action="store_false", + help="Don't serve the generated docs after building.", +) +@CommandArgument( + "--http", + default="localhost:5500", + metavar="ADDRESS", + help="Serve documentation on the specified host and port, " + 'default "localhost:5500".', +) +@CommandArgument("--upload", action="store_true", help="Upload generated files to S3.") +@CommandArgument( + "-j", + "--jobs", + default=str(multiprocessing.cpu_count()), + dest="jobs", + help="Distribute the build over N processes in parallel.", +) +@CommandArgument("--write-url", default=None, help="Write S3 Upload URL to text file") +@CommandArgument( + "--linkcheck", action="store_true", help="Check if the links are still valid" +) +@CommandArgument( + "--dump-trees", default=None, help="Dump the Sphinx trees to specified file." +) +@CommandArgument( + "--fatal-warnings", + dest="enable_fatal_warnings", + action="store_true", + help="Enable fatal warnings.", +) +@CommandArgument( + "--check-num-warnings", + action="store_true", + help="Check that the upper bound on the number of warnings is respected.", +) +@CommandArgument("--verbose", action="store_true", help="Run Sphinx in verbose mode") +@CommandArgument( + "--no-autodoc", + action="store_true", + help="Disable generating Python/JS API documentation", +) +def build_docs( + command_context, + path=None, + fmt="html", + outdir=None, + auto_open=True, + serve=True, + http=None, + archive=False, + upload=False, + jobs=None, + write_url=None, + linkcheck=None, + dump_trees=None, + enable_fatal_warnings=False, + check_num_warnings=False, + verbose=None, + no_autodoc=False, +): + # TODO: Bug 1704891 - move the ESLint setup tools to a shared place. + import setup_helper + + setup_helper.set_project_root(command_context.topsrcdir) + + if not setup_helper.check_node_executables_valid(): + return 1 + + setup_helper.eslint_maybe_setup() + + # Set the path so that Sphinx can find jsdoc, unfortunately there isn't + # a way to pass this to Sphinx itself at the moment. + os.environ["PATH"] = ( + mozpath.join(command_context.topsrcdir, "node_modules", ".bin") + + os.pathsep + + _node_path() + + os.pathsep + + os.environ["PATH"] + ) + + import webbrowser + + from livereload import Server + + from moztreedocs.package import create_tarball + + unique_id = "%s/%s" % (project(), str(uuid.uuid1())) + + outdir = outdir or os.path.join(command_context.topobjdir, "docs") + savedir = os.path.join(outdir, fmt) + + if path is None: + path = command_context.topsrcdir + if os.environ.get("MOZ_AUTOMATION") != "1": + print( + "\nBuilding the full documentation tree.\n" + "Did you mean to only build part of the documentation?\n" + "For a faster command, consider running:\n" + " ./mach doc path/to/docs\n" + ) + path = os.path.normpath(os.path.abspath(path)) + + docdir = _find_doc_dir(path) + if not docdir: + print(_dump_sphinx_backtrace()) + return die( + "failed to generate documentation:\n" + "%s: could not find docs at this location" % path + ) + + if linkcheck: + # We want to verify if the links are valid or not + fmt = "linkcheck" + if no_autodoc: + if check_num_warnings: + return die( + "'--no-autodoc' flag may not be used with '--check-num-warnings'" + ) + toggle_no_autodoc() + + status, warnings = _run_sphinx(docdir, savedir, fmt=fmt, jobs=jobs, verbose=verbose) + if status != 0: + print(_dump_sphinx_backtrace()) + return die( + "failed to generate documentation:\n" + "%s: sphinx return code %d" % (path, status) + ) + else: + print("\nGenerated documentation:\n%s" % savedir) + msg = "" + + if enable_fatal_warnings: + fatal_warnings = _check_sphinx_fatal_warnings(warnings) + if fatal_warnings: + msg += f"Error: Got fatal warnings:\n{''.join(fatal_warnings)}" + if check_num_warnings: + num_new = _check_sphinx_num_warnings(warnings) + if num_new: + msg += f"Error: {num_new} new warnings" + if msg: + return die(f"failed to generate documentation:\n {msg}") + + # Upload the artifact containing the link to S3 + # This would be used by code-review to post the link to Phabricator + if write_url is not None: + unique_link = BASE_LINK + unique_id + "/index.html" + with open(write_url, "w") as fp: + fp.write(unique_link) + fp.flush() + print("Generated " + write_url) + + if dump_trees is not None: + parent = os.path.dirname(dump_trees) + if parent and not os.path.isdir(parent): + os.makedirs(parent) + with open(dump_trees, "w") as fh: + json.dump(manager().trees, fh) + + if archive: + archive_path = os.path.join(outdir, "%s.tar.gz" % project()) + create_tarball(archive_path, savedir) + print("Archived to %s" % archive_path) + + if upload: + _s3_upload(savedir, project(), unique_id, version()) + + if not serve: + index_path = os.path.join(savedir, "index.html") + if auto_open and os.path.isfile(index_path): + webbrowser.open(index_path) + return + + # Create livereload server. Any files modified in the specified docdir + # will cause a re-build and refresh of the browser (if open). + try: + host, port = http.split(":", 1) + port = int(port) + except ValueError: + return die("invalid address: %s" % http) + + server = Server() + + sphinx_trees = manager().trees or {savedir: docdir} + for _, src in sphinx_trees.items(): + run_sphinx = partial( + _run_sphinx, src, savedir, fmt=fmt, jobs=jobs, verbose=verbose + ) + server.watch(src, run_sphinx) + server.serve( + host=host, + port=port, + root=savedir, + open_url_delay=0.1 if auto_open else None, + ) + + +def _dump_sphinx_backtrace(): + """ + If there is a sphinx dump file, read and return + its content. + By default, it isn't displayed. + """ + pattern = "sphinx-err-*" + output = "" + tmpdir = "/tmp" + + if not os.path.isdir(tmpdir): + # Only run it on Linux + return + files = os.listdir(tmpdir) + for name in files: + if fnmatch.fnmatch(name, pattern): + pathFile = os.path.join(tmpdir, name) + stat = os.stat(pathFile) + output += "Name: {0} / Creation date: {1}\n".format( + pathFile, time.ctime(stat.st_mtime) + ) + with open(pathFile) as f: + output += f.read() + return output + + +def _run_sphinx(docdir, savedir, config=None, fmt="html", jobs=None, verbose=None): + import sphinx.cmd.build + + config = config or manager().conf_py_path + # When running sphinx with sentry, it adds significant overhead + # and makes the build generation very very very slow + # So, disable it to generate the doc faster + sentry_sdk.init(None) + warn_fd, warn_path = tempfile.mkstemp() + os.close(warn_fd) + try: + args = [ + "-T", + "-b", + fmt, + "-c", + os.path.dirname(config), + "-w", + warn_path, + docdir, + savedir, + ] + if jobs: + args.extend(["-j", jobs]) + if verbose: + args.extend(["-v", "-v"]) + print("Run sphinx with:") + print(args) + status = sphinx.cmd.build.build_main(args) + with open(warn_path) as warn_file: + warnings = warn_file.readlines() + return status, warnings + finally: + try: + os.unlink(warn_path) + except Exception as ex: + print(ex) + + +def _check_sphinx_fatal_warnings(warnings): + with open(os.path.join(DOC_ROOT, "config.yml"), "r") as fh: + fatal_warnings_src = yaml.safe_load(fh)["fatal warnings"] + fatal_warnings_regex = [re.compile(item) for item in fatal_warnings_src] + fatal_warnings = [] + for warning in warnings: + if any(item.search(warning) for item in fatal_warnings_regex): + fatal_warnings.append(warning) + return fatal_warnings + + +def _check_sphinx_num_warnings(warnings): + # warnings file contains other strings as well + num_warnings = len([w for w in warnings if "WARNING" in w]) + with open(os.path.join(DOC_ROOT, "config.yml"), "r") as fh: + max_num = yaml.safe_load(fh)["max_num_warnings"] + if num_warnings > max_num: + return num_warnings - max_num + return None + + +def manager(): + from moztreedocs import manager + + return manager + + +def toggle_no_autodoc(): + import moztreedocs + + moztreedocs._SphinxManager.NO_AUTODOC = True + + +@memoize +def _read_project_properties(): + import imp + + path = os.path.normpath(manager().conf_py_path) + with open(path, "r") as fh: + conf = imp.load_module("doc_conf", fh, path, (".py", "r", imp.PY_SOURCE)) + + # Prefer the Mozilla project name, falling back to Sphinx's + # default variable if it isn't defined. + project = getattr(conf, "moz_project_name", None) + if not project: + project = conf.project.replace(" ", "_") + + return {"project": project, "version": getattr(conf, "version", None)} + + +def project(): + return _read_project_properties()["project"] + + +def version(): + return _read_project_properties()["version"] + + +def _node_path(): + from mozbuild.nodeutil import find_node_executable + + node, _ = find_node_executable() + + return os.path.dirname(node) + + +def _find_doc_dir(path): + if os.path.isfile(path): + return + + valid_doc_dirs = ("doc", "docs") + for d in valid_doc_dirs: + p = os.path.join(path, d) + if os.path.isdir(p): + path = p + + for index_file in ["index.rst", "index.md"]: + if os.path.exists(os.path.join(path, index_file)): + return path + + +def _s3_upload(root, project, unique_id, version=None): + # Workaround the issue + # BlockingIOError: [Errno 11] write could not complete without blocking + # https://github.com/travis-ci/travis-ci/issues/8920 + import fcntl + + from moztreedocs.package import distribution_files + from moztreedocs.upload import s3_set_redirects, s3_upload + + fcntl.fcntl(1, fcntl.F_SETFL, 0) + + # Files are uploaded to multiple locations: + # + # <project>/latest + # <project>/<version> + # + # This allows multiple projects and versions to be stored in the + # S3 bucket. + + files = list(distribution_files(root)) + key_prefixes = [] + if version: + key_prefixes.append("%s/%s" % (project, version)) + + # Until we redirect / to main/latest, upload the main docs + # to the root. + if project == "main": + key_prefixes.append("") + + key_prefixes.append(unique_id) + + with open(os.path.join(DOC_ROOT, "config.yml"), "r") as fh: + redirects = yaml.safe_load(fh)["redirects"] + + redirects = {k.strip("/"): v.strip("/") for k, v in redirects.items()} + + all_redirects = {} + + for prefix in key_prefixes: + s3_upload(files, prefix) + + # Don't setup redirects for the "version" or "uuid" prefixes since + # we are exceeding a 50 redirect limit and external things are + # unlikely to link there anyway (see bug 1614908). + if (version and prefix.endswith(version)) or prefix == unique_id: + continue + + if prefix: + prefix += "/" + all_redirects.update({prefix + k: prefix + v for k, v in redirects.items()}) + + print("Redirects currently staged") + pprint(all_redirects, indent=1) + + s3_set_redirects(all_redirects) + + unique_link = BASE_LINK + unique_id + "/index.html" + print("Uploaded documentation can be accessed here " + unique_link) + + +@SubCommand( + "doc", + "mach-telemetry", + description="Generate documentation from Glean metrics.yaml files", +) +def generate_telemetry_docs(command_context): + args = [ + sys.executable, + "-m" "glean_parser", + "translate", + "-f", + "markdown", + "-o", + os.path.join(topsrcdir, "python/mach/docs/"), + os.path.join(topsrcdir, "python/mach/pings.yaml"), + os.path.join(topsrcdir, "python/mach/metrics.yaml"), + ] + metrics_paths = [ + handler.metrics_path + for handler in Registrar.command_handlers.values() + if handler.metrics_path is not None + ] + args.extend( + [os.path.join(command_context.topsrcdir, path) for path in set(metrics_paths)] + ) + subprocess.check_call(args) + + +@SubCommand( + "doc", + "show-targets", + description="List all reference targets. Requires the docs to have been built.", +) +@CommandArgument( + "--format", default="html", dest="fmt", help="Documentation format used." +) +@CommandArgument( + "--outdir", default=None, metavar="DESTINATION", help="Where output was written." +) +def show_reference_targets(command_context, fmt="html", outdir=None): + command_context.activate_virtualenv() + command_context.virtualenv_manager.install_pip_requirements( + os.path.join(here, "requirements.txt") + ) + + import sphinx.ext.intersphinx + + outdir = outdir or os.path.join(command_context.topobjdir, "docs") + inv_path = os.path.join(outdir, fmt, "objects.inv") + + if not os.path.exists(inv_path): + return die( + "object inventory not found: {inv_path}.\n" + "Rebuild the docs and rerun this command" + ) + sphinx.ext.intersphinx.inspect_main([inv_path]) + + +def die(msg, exit_code=1): + msg = "%s: %s" % (sys.argv[0], msg) + print(msg, file=sys.stderr) + return exit_code |