diff options
Diffstat (limited to 'tools/moztreedocs')
| -rw-r--r-- | tools/moztreedocs/__init__.py | 208 | ||||
| -rw-r--r-- | tools/moztreedocs/docs/adding-documentation.rst | 30 | ||||
| -rw-r--r-- | tools/moztreedocs/docs/architecture.rst | 51 | ||||
| -rw-r--r-- | tools/moztreedocs/docs/index.rst | 24 | ||||
| -rw-r--r-- | tools/moztreedocs/docs/jsdoc-support.rst | 16 | ||||
| -rw-r--r-- | tools/moztreedocs/docs/mdn-import.rst | 34 | ||||
| -rw-r--r-- | tools/moztreedocs/docs/mermaid-integration.rst | 104 | ||||
| -rw-r--r-- | tools/moztreedocs/docs/nested-docs.rst | 14 | ||||
| -rw-r--r-- | tools/moztreedocs/docs/redirect.rst | 11 | ||||
| -rw-r--r-- | tools/moztreedocs/docs/rstlint.rst | 12 | ||||
| -rw-r--r-- | tools/moztreedocs/docs/run-try-job.rst | 27 | ||||
| -rw-r--r-- | tools/moztreedocs/docs/server-synchronization.rst | 5 | ||||
| -rw-r--r-- | tools/moztreedocs/mach_commands.py | 539 | ||||
| -rw-r--r-- | tools/moztreedocs/package.py | 29 | ||||
| -rw-r--r-- | tools/moztreedocs/upload.py | 171 |
15 files changed, 1275 insertions, 0 deletions
diff --git a/tools/moztreedocs/__init__.py b/tools/moztreedocs/__init__.py new file mode 100644 index 0000000000..7b8aed6059 --- /dev/null +++ b/tools/moztreedocs/__init__.py @@ -0,0 +1,208 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, # You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +from pathlib import PurePath + +import sphinx +import sphinx.ext.apidoc +import yaml +from mozbuild.base import MozbuildObject +from mozbuild.frontend.reader import BuildReader +from mozbuild.util import memoize +from mozpack.copier import FileCopier +from mozpack.files import FileFinder +from mozpack.manifests import InstallManifest + +here = os.path.abspath(os.path.dirname(__file__)) +build = MozbuildObject.from_environment(cwd=here) + +MAIN_DOC_PATH = os.path.normpath(os.path.join(build.topsrcdir, "docs")) + +logger = sphinx.util.logging.getLogger(__name__) + + +@memoize +def read_build_config(docdir): + """Read the active build config and return the relevant doc paths. + + The return value is cached so re-generating with the same docdir won't + invoke the build system a second time.""" + trees = {} + python_package_dirs = set() + + is_main = docdir == MAIN_DOC_PATH + relevant_mozbuild_path = None if is_main else docdir + + # Reading the Sphinx variables doesn't require a full build context. + # Only define the parts we need. + class fakeconfig(object): + topsrcdir = build.topsrcdir + + variables = ("SPHINX_TREES", "SPHINX_PYTHON_PACKAGE_DIRS") + reader = BuildReader(fakeconfig()) + result = reader.find_variables_from_ast(variables, path=relevant_mozbuild_path) + for path, name, key, value in result: + reldir = os.path.dirname(path) + + if name == "SPHINX_TREES": + # If we're building a subtree, only process that specific subtree. + # topsrcdir always uses POSIX-style path, normalize it for proper comparison. + absdir = os.path.normpath(os.path.join(build.topsrcdir, reldir, value)) + if not is_main and absdir not in (docdir, MAIN_DOC_PATH): + # allow subpaths of absdir (i.e. docdir = <absdir>/sub/path/) + if docdir.startswith(absdir): + key = os.path.join(key, docdir.split(f"{key}/")[-1]) + else: + continue + + assert key + if key.startswith("/"): + key = key[1:] + else: + key = os.path.normpath(os.path.join(reldir, key)) + + if key in trees: + raise Exception( + "%s has already been registered as a destination." % key + ) + trees[key] = os.path.join(reldir, value) + + if name == "SPHINX_PYTHON_PACKAGE_DIRS": + python_package_dirs.add(os.path.join(reldir, value)) + + return trees, python_package_dirs + + +class _SphinxManager(object): + """Manages the generation of Sphinx documentation for the tree.""" + + NO_AUTODOC = False + + def __init__(self, topsrcdir, main_path): + self.topsrcdir = topsrcdir + self.conf_py_path = os.path.join(main_path, "conf.py") + self.index_path = os.path.join(main_path, "index.rst") + + # Instance variables that get set in self.generate_docs() + self.staging_dir = None + self.trees = None + self.python_package_dirs = None + + def generate_docs(self, app): + """Generate/stage documentation.""" + if self.NO_AUTODOC: + logger.info("Python/JS API documentation generation will be skipped") + app.config["extensions"].remove("sphinx.ext.autodoc") + app.config["extensions"].remove("sphinx_js") + self.staging_dir = os.path.join(app.outdir, "_staging") + + logger.info("Reading Sphinx metadata from build configuration") + self.trees, self.python_package_dirs = read_build_config(app.srcdir) + + logger.info("Staging static documentation") + self._synchronize_docs(app) + + if not self.NO_AUTODOC: + self._generate_python_api_docs() + + def _generate_python_api_docs(self): + """Generate Python API doc files.""" + out_dir = os.path.join(self.staging_dir, "python") + base_args = ["--no-toc", "-o", out_dir] + + for p in sorted(self.python_package_dirs): + full = os.path.join(self.topsrcdir, p) + + finder = FileFinder(full) + dirs = {os.path.dirname(f[0]) for f in finder.find("**")} + + test_dirs = {"test", "tests"} + excludes = {d for d in dirs if set(PurePath(d).parts) & test_dirs} + + args = list(base_args) + args.append(full) + args.extend(excludes) + + sphinx.ext.apidoc.main(argv=args) + + def _synchronize_docs(self, app): + m = InstallManifest() + + with open(os.path.join(MAIN_DOC_PATH, "config.yml"), "r") as fh: + tree_config = yaml.safe_load(fh)["categories"] + + m.add_link(self.conf_py_path, "conf.py") + + for dest, source in sorted(self.trees.items()): + source_dir = os.path.join(self.topsrcdir, source) + for root, _, files in os.walk(source_dir): + for f in files: + source_path = os.path.normpath(os.path.join(root, f)) + rel_source = source_path[len(source_dir) + 1 :] + target = os.path.normpath(os.path.join(dest, rel_source)) + m.add_link(source_path, target) + + copier = FileCopier() + m.populate_registry(copier) + + # In the case of livereload, we don't want to delete unmodified (unaccounted) files. + copier.copy( + self.staging_dir, remove_empty_directories=False, remove_unaccounted=False + ) + + with open(self.index_path, "r") as fh: + data = fh.read() + + def is_toplevel(key): + """Whether the tree is nested under the toplevel index, or is + nested under another tree's index. + """ + for k in self.trees: + if k == key: + continue + if key.startswith(k): + return False + return True + + def format_paths(paths): + source_doc = ["%s/index" % p for p in paths] + return "\n ".join(source_doc) + + toplevel_trees = {k: v for k, v in self.trees.items() if is_toplevel(k)} + + CATEGORIES = {} + # generate the datastructure to deal with the tree + for t in tree_config: + CATEGORIES[t] = format_paths(tree_config[t]) + + # During livereload, we don't correctly rebuild the full document + # tree (Bug 1557020). The page is no longer referenced within the index + # tree, thus we shall check categorisation only if complete tree is being rebuilt. + if app.srcdir == self.topsrcdir: + indexes = set( + [ + os.path.normpath(os.path.join(p, "index")) + for p in toplevel_trees.keys() + ] + ) + # Format categories like indexes + cats = "\n".join(CATEGORIES.values()).split("\n") + # Remove heading spaces + cats = [os.path.normpath(x.strip()) for x in cats] + indexes = tuple(set(indexes) - set(cats)) + if indexes: + # In case a new doc isn't categorized + print(indexes) + raise Exception( + "Uncategorized documentation. Please add it in docs/config.yml" + ) + + data = data.format(**CATEGORIES) + + with open(os.path.join(self.staging_dir, "index.rst"), "w") as fh: + fh.write(data) + + +manager = _SphinxManager(build.topsrcdir, MAIN_DOC_PATH) diff --git a/tools/moztreedocs/docs/adding-documentation.rst b/tools/moztreedocs/docs/adding-documentation.rst new file mode 100644 index 0000000000..9abb6a2f84 --- /dev/null +++ b/tools/moztreedocs/docs/adding-documentation.rst @@ -0,0 +1,30 @@ +Adding Documentation +-------------------- + +To add new documentation, define the ``SPHINX_TREES`` and +``SPHINX_PYTHON_PACKAGE_DIRS`` variables in ``moz.build`` files in +the tree and documentation will automatically get picked up. + +Say you have a directory ``featureX`` you would like to write some +documentation for. Here are the steps to create Sphinx documentation +for it: + +1. Create a directory for the docs. This is typically ``docs``. e.g. + ``featureX/docs``. +2. Create an ``index.rst`` file in this directory. The ``index.rst`` file + is the root documentation for that section. See ``build/docs/index.rst`` + for an example file. +3. In a ``moz.build`` file (typically the one in the parent directory of + the ``docs`` directory), define ``SPHINX_TREES`` to hook up the plumbing. + e.g. ``SPHINX_TREES['featureX'] = 'docs'``. This says *the ``docs`` + directory under the current directory should be installed into the + Sphinx documentation tree under ``/featureX``*. +4. If you have Python packages you would like to generate Python API + documentation for, you can use ``SPHINX_PYTHON_PACKAGE_DIRS`` to + declare directories containing Python packages. e.g. + ``SPHINX_PYTHON_PACKAGE_DIRS += ['mozpackage']``. +5. In ``docs/config.yml``, defines in which category the doc + should go. +6. Verify the rst syntax using `./mach lint -l rst`_ + +.. _./mach lint -l rst: /tools/lint/linters/rstlinter.html diff --git a/tools/moztreedocs/docs/architecture.rst b/tools/moztreedocs/docs/architecture.rst new file mode 100644 index 0000000000..fc502f847f --- /dev/null +++ b/tools/moztreedocs/docs/architecture.rst @@ -0,0 +1,51 @@ +Documentation architecture +========================== + +The documentation relies on Sphinx and many Sphinx extensions. + +The documentation code is in two main directories: + +* https://searchfox.org/mozilla-central/source/docs +* https://searchfox.org/mozilla-central/source/tools/moztreedocs + +Our documentation supports both rst & markdown syntaxes. + +Configuration +------------- + +The main configuration file is: + +https://searchfox.org/mozilla-central/source/docs/config.yml + +It contains the categories, the redirects, the warnings and others configuration aspects. + +The dependencies are listed in: + +https://searchfox.org/mozilla-central/source/tools/moztreedocs/requirements.in + +Be aware that Python libraries stored in `third_party/python` are used in priority (not always for good reasons). See :ref:`Vendor the source of the Python package in-tree <python-vendor>` for more details. + + +Architecture +------------ + + +`mach_commands <https://searchfox.org/mozilla-central/source/tools/moztreedocs/mach_commands.py>`__ +contains: + +* `mach doc` arguments managements +* Detection/configuration of the environment (nodejs for jsdoc, pip for dependencies, etc) +* Symlink the doc sources (.rst & .md) from the source tree into the staging directory +* Fails the build if any critical warnings have been identified +* Starts the sphinx build (and serve it if the option is set) +* Manages telemetry + +`docs/conf.py <https://searchfox.org/mozilla-central/source/docs/conf.py>`__ defines: + +* The list of extensions +* JS source paths +* Various sphinx configuration + +At the end of the build documentation process, files will be uploaded to a CDN: + +https://searchfox.org/mozilla-central/source/tools/moztreedocs/upload.py diff --git a/tools/moztreedocs/docs/index.rst b/tools/moztreedocs/docs/index.rst new file mode 100644 index 0000000000..fc03756fbe --- /dev/null +++ b/tools/moztreedocs/docs/index.rst @@ -0,0 +1,24 @@ +Managing Documentation +====================== + +Documentation is hard. It's difficult to write, difficult to find and always out +of date. That's why we implemented our in-tree documentation system that +underpins firefox-source-docs.mozilla.org. The documentation lives next to the +code that it documents, so it can be updated within the same commit that makes +the underlying changes. + +This documentation is generated via the +`Sphinx <http://sphinx-doc.org/>`_ tool from sources in the tree. + +To build the documentation, run ``mach doc``. Run +``mach help doc`` to see configurable options. + +The review group in Phabricator is ``#firefox-source-docs-reviewers``. +For simple documentation changes, reviews are not required. + +.. toctree:: + :caption: Documentation + :maxdepth: 2 + :glob: + + * diff --git a/tools/moztreedocs/docs/jsdoc-support.rst b/tools/moztreedocs/docs/jsdoc-support.rst new file mode 100644 index 0000000000..100fb92dac --- /dev/null +++ b/tools/moztreedocs/docs/jsdoc-support.rst @@ -0,0 +1,16 @@ +jsdoc support +============= + +Here is a quick example, for the public AddonManager :ref:`API <AddonManager Reference>` + +To use it for your own code: + +#. Check that JSDoc generates the output you expect (you may need to use a @class annotation on "object initializer"-style class definitions for instance) + +#. Create an `.rst file`, which may contain explanatory text as well as the API docs. The minimum will look something like + `this <https://firefox-source-docs.mozilla.org/_sources/toolkit/mozapps/extensions/addon-manager/AddonManager.rst.txt>`__ + +#. Ensure your component is on the js_source_path here in the sphinx + config: https://hg.mozilla.org/mozilla-central/file/72ee4800d415/tools/docs/conf.py#l46 + +#. Run `mach doc` locally to generate the output and confirm that it looks correct. diff --git a/tools/moztreedocs/docs/mdn-import.rst b/tools/moztreedocs/docs/mdn-import.rst new file mode 100644 index 0000000000..69d2f56df4 --- /dev/null +++ b/tools/moztreedocs/docs/mdn-import.rst @@ -0,0 +1,34 @@ +Importing documentation from MDN +-------------------------------- + +As MDN should not be used for documenting mozilla-central specific code or process, +the documentation should be migrated in this repository. + +The meta bug is `Bug 1617963 <https://bugzilla.mozilla.org/show_bug.cgi?id=migrate-from-mdn>`__. + +Fortunately, there is an easy way to import the doc from MDN using GitHub +to the firefox source docs. + +1. Install https://pandoc.org/ - If you are using packages provided by your distribution, + make sure that the version is not too old. + +2. Identify where your page is located on the GitHub repository ( https://github.com/mdn/archived-content/tree/main/files/en-us/mozilla ). + Get the raw URL + +3. Run pandoc the following way: + +.. code-block:: shell + + $ pandoc -t rst https://github.com/mdn/archived-content/tree/main/files/en-us/mozilla/firefox/performance_best_practices_for_firefox_fe_engineers > doc.rst + +4. In the new doc.rst, identify the images and wget/curl them into `img/`. + +5. Verify the rst syntax using `./mach lint -l rst`_ + +.. _./mach lint -l rst: /tools/lint/linters/rstlinter.html + +6. If relevant, remove unbreakable spaces (rendered with a "!" on Phabricator) + +.. code-block:: shell + + $ sed -i -e 's/\xc2\xa0/ /g' doc.rst diff --git a/tools/moztreedocs/docs/mermaid-integration.rst b/tools/moztreedocs/docs/mermaid-integration.rst new file mode 100644 index 0000000000..ec0da3dd74 --- /dev/null +++ b/tools/moztreedocs/docs/mermaid-integration.rst @@ -0,0 +1,104 @@ +Mermaid Integration +=================== + +Mermaid is a tool that lets you generate flow charts, sequence diagrams, gantt +charts, class diagrams and vcs graphs from a simple markup language. This +allows charts and diagrams to be embedded and edited directly in the +documentation source files rather than creating them as images using some +external tool and checking the images into the tree. + +To add a diagram, simply put something like this into your page: + +.. These two examples come from the upstream website (https://mermaid-js.github.io/mermaid/#/) + +.. code-block:: rst + :caption: .rst + + .. mermaid:: + + graph TD; + A-->B; + A-->C; + B-->D; + C-->D; + +.. code-block:: md + :caption: .md + + ```{mermaid} + graph TD; + A-->B; + A-->C; + B-->D; + C-->D; + ``` + +The result will be: + +.. mermaid:: + + graph TD; + A-->B; + A-->C; + B-->D; + C-->D; + +Or + +.. code-block:: rst + :caption: .rst + + .. mermaid:: + + sequenceDiagram + participant Alice + participant Bob + Alice->>John: Hello John, how are you? + loop Healthcheck + John->>John: Fight against hypochondria + end + Note right of John: Rational thoughts <br/>prevail! + John-->>Alice: Great! + John->>Bob: How about you? + Bob-->>John: Jolly good! + +.. code-block:: markdown + :caption: .md + + ```{mermaid} + sequenceDiagram + participant Alice + participant Bob + Alice->>John: Hello John, how are you? + loop Healthcheck + John->>John: Fight against hypochondria + end + Note right of John: Rational thoughts <br/>prevail! + John-->>Alice: Great! + John->>Bob: How about you? + Bob-->>John: Jolly good! + ``` + + + +will show: + +.. mermaid:: + + sequenceDiagram + participant Alice + participant Bob + Alice->>John: Hello John, how are you? + loop Healthcheck + John->>John: Fight against hypochondria + end + Note right of John: Rational thoughts <br/>prevail! + John-->>Alice: Great! + John->>Bob: How about you? + Bob-->>John: Jolly good! + + +See `Mermaid's official <https://mermaid-js.github.io/mermaid/#/>`__ docs for +more details on the syntax, and use the +`Mermaid Live Editor <https://mermaidjs.github.io/mermaid-live-editor/>`__ to +experiment with creating your own diagrams. diff --git a/tools/moztreedocs/docs/nested-docs.rst b/tools/moztreedocs/docs/nested-docs.rst new file mode 100644 index 0000000000..e2eb03b42d --- /dev/null +++ b/tools/moztreedocs/docs/nested-docs.rst @@ -0,0 +1,14 @@ +Nested Doc Trees +================ + +This feature essentially means we can now group related docs together under +common "landing pages". This will allow us to refactor the docs into a structure that makes more sense. For example we could have a landing page for docs describing Gecko's internals, and another one for docs describing developer workflows in `mozilla-central`. + + +To clarify a few things: + +#. The path specified in `SPHINX_TREES` does not need to correspond to a path in `mozilla-central`. For example, I could register my docs using `SPHINX_TREES["/foo"] = "docs"`, which would make that doc tree accessible at `firefox-source-docs.mozilla.org/foo`. + +#. Any subtrees that are nested under another index will automatically be hidden from the main index. This means you should make sure to link to any subtrees from somewhere in the landing page. So given my earlier doc tree at `/foo`, if I now created a subtree and registered it using `SPHINX_TREES["/foo/bar"] = "docs"`, those docs would not show up in the main index. + +#. The relation between subtrees and their parents does not necessarily have any bearing with their relation on the file system. For example, a doc tree that lives under `/devtools` can be nested under an index that lives under `/browser`. diff --git a/tools/moztreedocs/docs/redirect.rst b/tools/moztreedocs/docs/redirect.rst new file mode 100644 index 0000000000..6ec29cdfd0 --- /dev/null +++ b/tools/moztreedocs/docs/redirect.rst @@ -0,0 +1,11 @@ +Redirects +========= + +We now have the ability to define redirects in-tree! This will allow us to +refactor and move docs around to our hearts content without needing to worry +about stale external URLs. To set up a redirect simply add a line to this file under ``redirects`` key: + +https://searchfox.org/mozilla-central/source/docs/config.yml + +Any request starting with the prefix on the left, will be rewritten to the prefix on the right by the server. So for example a request to +``/testing/marionette/marionette/index.html`` will be re-written to ``/testing/marionette/index.html``. Amazon's API only supports prefix redirects, so anything more complex isn't supported. diff --git a/tools/moztreedocs/docs/rstlint.rst b/tools/moztreedocs/docs/rstlint.rst new file mode 100644 index 0000000000..230ba2e812 --- /dev/null +++ b/tools/moztreedocs/docs/rstlint.rst @@ -0,0 +1,12 @@ +ReStructuredText Linter +----------------------- + +RST isn't the easiest of markup languages, but it's powerful and what `Sphinx` (the library used to build our docs) uses, so we're stuck with it. But at least we now have a linter which will catch basic problems in `.rst` files early. Be sure to run: + +.. code-block:: shell + + mach lint -l rst + +to test your outgoing changes before submitting to review. + +`More information <RST Linter>`__. diff --git a/tools/moztreedocs/docs/run-try-job.rst b/tools/moztreedocs/docs/run-try-job.rst new file mode 100644 index 0000000000..d7fe6b20b8 --- /dev/null +++ b/tools/moztreedocs/docs/run-try-job.rst @@ -0,0 +1,27 @@ +Running a try job for Documentation +----------------------------------- + +Documentation has two try jobs associated: + + - ``doc-generate`` - This generates the documentation with the committed changes on the try server and gives the same output as if it has landed on regular integration branch. + + .. code-block:: shell + + mach try fuzzy -q "'doc-generate" + + - ``doc-upload`` - This uploads documentation to `gecko-l1 bucket <http://gecko-docs.mozilla.org-l1.s3.us-west-2.amazonaws.com/index.html>`__ with the committed changes. + + .. code-block:: shell + + mach try fuzzy -q "'doc-upload" + +When the documentation is modified, at review phase, reviewbot will automatically generate a temporary documentation with a direct link to the modified pages. + +.. important:: + + Running try jobs require the user to have try server access. + +.. note:: + + To learn more about setting up try server or + using a different selector head over to :ref:`try server documentation <Pushing to Try>` diff --git a/tools/moztreedocs/docs/server-synchronization.rst b/tools/moztreedocs/docs/server-synchronization.rst new file mode 100644 index 0000000000..b47b66503e --- /dev/null +++ b/tools/moztreedocs/docs/server-synchronization.rst @@ -0,0 +1,5 @@ +Server Synchronization +====================== + +We now compare all the files that exist on the server against the list of source files in `mozilla-central`. +Any files on the server that no longer exist in `mozilla-central` are removed. diff --git a/tools/moztreedocs/mach_commands.py b/tools/moztreedocs/mach_commands.py new file mode 100644 index 0000000000..ad63ae0e44 --- /dev/null +++ b/tools/moztreedocs/mach_commands.py @@ -0,0 +1,539 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, # You can obtain one at http://mozilla.org/MPL/2.0/. + +import fnmatch +import json +import multiprocessing +import os +import re +import subprocess +import sys +import tempfile +import time +import uuid +from functools import partial +from pprint import pprint + +import mozpack.path as mozpath +import sentry_sdk +import yaml +from mach.decorators import Command, CommandArgument, SubCommand +from mach.registrar import Registrar +from mozbuild.util import memoize +from mozfile import load_source + +here = os.path.abspath(os.path.dirname(__file__)) +topsrcdir = os.path.abspath(os.path.dirname(os.path.dirname(here))) +DOC_ROOT = os.path.join(topsrcdir, "docs") +BASE_LINK = "http://gecko-docs.mozilla.org-l1.s3-website.us-west-2.amazonaws.com/" + + +# Helps manage in-tree documentation. + + +@Command( + "doc", + category="devenv", + virtualenv_name="docs", + description="Generate and serve documentation from the tree.", +) +@CommandArgument( + "path", + default=None, + metavar="DIRECTORY", + nargs="?", + help="Path to documentation to build and display.", +) +@CommandArgument( + "--format", default="html", dest="fmt", help="Documentation format to write." +) +@CommandArgument( + "--outdir", default=None, metavar="DESTINATION", help="Where to write output." +) +@CommandArgument( + "--archive", + action="store_true", + help="Write a gzipped tarball of generated docs.", +) +@CommandArgument( + "--no-open", + dest="auto_open", + default=True, + action="store_false", + help="Don't automatically open HTML docs in a browser.", +) +@CommandArgument( + "--no-serve", + dest="serve", + default=True, + action="store_false", + help="Don't serve the generated docs after building.", +) +@CommandArgument( + "--http", + default="localhost:5500", + metavar="ADDRESS", + help="Serve documentation on the specified host and port, " + 'default "localhost:5500".', +) +@CommandArgument("--upload", action="store_true", help="Upload generated files to S3.") +@CommandArgument( + "-j", + "--jobs", + default=str(multiprocessing.cpu_count()), + dest="jobs", + help="Distribute the build over N processes in parallel.", +) +@CommandArgument("--write-url", default=None, help="Write S3 Upload URL to text file") +@CommandArgument( + "--linkcheck", action="store_true", help="Check if the links are still valid" +) +@CommandArgument( + "--dump-trees", default=None, help="Dump the Sphinx trees to specified file." +) +@CommandArgument( + "--fatal-warnings", + dest="enable_fatal_warnings", + action="store_true", + help="Enable fatal warnings.", +) +@CommandArgument( + "--check-num-warnings", + action="store_true", + help="Check that the upper bound on the number of warnings is respected.", +) +@CommandArgument("--verbose", action="store_true", help="Run Sphinx in verbose mode") +@CommandArgument( + "--no-autodoc", + action="store_true", + help="Disable generating Python/JS API documentation", +) +def build_docs( + command_context, + path=None, + fmt="html", + outdir=None, + auto_open=True, + serve=True, + http=None, + archive=False, + upload=False, + jobs=None, + write_url=None, + linkcheck=None, + dump_trees=None, + enable_fatal_warnings=False, + check_num_warnings=False, + verbose=None, + no_autodoc=False, +): + # TODO: Bug 1704891 - move the ESLint setup tools to a shared place. + import setup_helper + + setup_helper.set_project_root(command_context.topsrcdir) + + if not setup_helper.check_node_executables_valid(): + return 1 + + setup_helper.eslint_maybe_setup() + + # Set the path so that Sphinx can find jsdoc, unfortunately there isn't + # a way to pass this to Sphinx itself at the moment. + os.environ["PATH"] = ( + mozpath.join(command_context.topsrcdir, "node_modules", ".bin") + + os.pathsep + + _node_path() + + os.pathsep + + os.environ["PATH"] + ) + + import webbrowser + + from livereload import Server + + from moztreedocs.package import create_tarball + + unique_id = "%s/%s" % (project(), str(uuid.uuid1())) + + outdir = outdir or os.path.join(command_context.topobjdir, "docs") + savedir = os.path.join(outdir, fmt) + + if path is None: + path = command_context.topsrcdir + if os.environ.get("MOZ_AUTOMATION") != "1": + print( + "\nBuilding the full documentation tree.\n" + "Did you mean to only build part of the documentation?\n" + "For a faster command, consider running:\n" + " ./mach doc path/to/docs\n" + ) + path = os.path.normpath(os.path.abspath(path)) + + docdir = _find_doc_dir(path) + if not docdir: + print(_dump_sphinx_backtrace()) + return die( + "failed to generate documentation:\n" + "%s: could not find docs at this location" % path + ) + + if linkcheck: + # We want to verify if the links are valid or not + fmt = "linkcheck" + if no_autodoc: + if check_num_warnings: + return die( + "'--no-autodoc' flag may not be used with '--check-num-warnings'" + ) + toggle_no_autodoc() + + status, warnings = _run_sphinx(docdir, savedir, fmt=fmt, jobs=jobs, verbose=verbose) + if status != 0: + print(_dump_sphinx_backtrace()) + return die( + "failed to generate documentation:\n" + "%s: sphinx return code %d" % (path, status) + ) + else: + print("\nGenerated documentation:\n%s" % savedir) + msg = "" + + if enable_fatal_warnings: + fatal_warnings = _check_sphinx_fatal_warnings(warnings) + if fatal_warnings: + msg += f"Error: Got fatal warnings:\n{''.join(fatal_warnings)}" + if check_num_warnings: + [num_new, num_actual] = _check_sphinx_num_warnings(warnings) + print("Logged %s warnings\n" % num_actual) + if num_new: + msg += f"Error: {num_new} new warnings have been introduced compared to the limit in docs/config.yml" + if msg: + return dieWithTestFailure(msg) + + # Upload the artifact containing the link to S3 + # This would be used by code-review to post the link to Phabricator + if write_url is not None: + unique_link = BASE_LINK + unique_id + "/index.html" + with open(write_url, "w") as fp: + fp.write(unique_link) + fp.flush() + print("Generated " + write_url) + + if dump_trees is not None: + parent = os.path.dirname(dump_trees) + if parent and not os.path.isdir(parent): + os.makedirs(parent) + with open(dump_trees, "w") as fh: + json.dump(manager().trees, fh) + + if archive: + archive_path = os.path.join(outdir, "%s.tar.gz" % project()) + create_tarball(archive_path, savedir) + print("Archived to %s" % archive_path) + + if upload: + _s3_upload(savedir, project(), unique_id, version()) + + if not serve: + index_path = os.path.join(savedir, "index.html") + if auto_open and os.path.isfile(index_path): + webbrowser.open(index_path) + return + + # Create livereload server. Any files modified in the specified docdir + # will cause a re-build and refresh of the browser (if open). + try: + host, port = http.split(":", 1) + port = int(port) + except ValueError: + return die("invalid address: %s" % http) + + server = Server() + + sphinx_trees = manager().trees or {savedir: docdir} + for _, src in sphinx_trees.items(): + run_sphinx = partial( + _run_sphinx, src, savedir, fmt=fmt, jobs=jobs, verbose=verbose + ) + server.watch(src, run_sphinx) + server.serve( + host=host, + port=port, + root=savedir, + open_url_delay=0.1 if auto_open else None, + ) + + +def _dump_sphinx_backtrace(): + """ + If there is a sphinx dump file, read and return + its content. + By default, it isn't displayed. + """ + pattern = "sphinx-err-*" + output = "" + tmpdir = "/tmp" + + if not os.path.isdir(tmpdir): + # Only run it on Linux + return + files = os.listdir(tmpdir) + for name in files: + if fnmatch.fnmatch(name, pattern): + pathFile = os.path.join(tmpdir, name) + stat = os.stat(pathFile) + output += "Name: {0} / Creation date: {1}\n".format( + pathFile, time.ctime(stat.st_mtime) + ) + with open(pathFile) as f: + output += f.read() + return output + + +def _run_sphinx(docdir, savedir, config=None, fmt="html", jobs=None, verbose=None): + import sphinx.cmd.build + + config = config or manager().conf_py_path + # When running sphinx with sentry, it adds significant overhead + # and makes the build generation very very very slow + # So, disable it to generate the doc faster + sentry_sdk.init(None) + warn_fd, warn_path = tempfile.mkstemp() + os.close(warn_fd) + try: + args = [ + "-T", + "-b", + fmt, + "-c", + os.path.dirname(config), + "-w", + warn_path, + docdir, + savedir, + ] + if jobs: + args.extend(["-j", jobs]) + if verbose: + args.extend(["-v", "-v"]) + print("Run sphinx with:") + print(args) + status = sphinx.cmd.build.build_main(args) + with open(warn_path) as warn_file: + warnings = warn_file.readlines() + return status, warnings + finally: + try: + os.unlink(warn_path) + except Exception as ex: + print(ex) + + +def _check_sphinx_fatal_warnings(warnings): + with open(os.path.join(DOC_ROOT, "config.yml"), "r") as fh: + fatal_warnings_src = yaml.safe_load(fh)["fatal warnings"] + fatal_warnings_regex = [re.compile(item) for item in fatal_warnings_src] + fatal_warnings = [] + for warning in warnings: + if any(item.search(warning) for item in fatal_warnings_regex): + fatal_warnings.append(warning) + return fatal_warnings + + +def _check_sphinx_num_warnings(warnings): + # warnings file contains other strings as well + num_warnings = len([w for w in warnings if "WARNING" in w]) + with open(os.path.join(DOC_ROOT, "config.yml"), "r") as fh: + max_num = yaml.safe_load(fh)["max_num_warnings"] + if num_warnings > max_num: + return [num_warnings - max_num, num_warnings] + return [0, num_warnings] + + +def manager(): + from moztreedocs import manager + + return manager + + +def toggle_no_autodoc(): + import moztreedocs + + moztreedocs._SphinxManager.NO_AUTODOC = True + + +@memoize +def _read_project_properties(): + path = os.path.normpath(manager().conf_py_path) + conf = load_source("doc_conf", path) + + # Prefer the Mozilla project name, falling back to Sphinx's + # default variable if it isn't defined. + project = getattr(conf, "moz_project_name", None) + if not project: + project = conf.project.replace(" ", "_") + + return {"project": project, "version": getattr(conf, "version", None)} + + +def project(): + return _read_project_properties()["project"] + + +def version(): + return _read_project_properties()["version"] + + +def _node_path(): + from mozbuild.nodeutil import find_node_executable + + node, _ = find_node_executable() + + return os.path.dirname(node) + + +def _find_doc_dir(path): + if os.path.isfile(path): + return + + valid_doc_dirs = ("doc", "docs") + for d in valid_doc_dirs: + p = os.path.join(path, d) + if os.path.isdir(p): + path = p + + for index_file in ["index.rst", "index.md"]: + if os.path.exists(os.path.join(path, index_file)): + return path + + +def _s3_upload(root, project, unique_id, version=None): + # Workaround the issue + # BlockingIOError: [Errno 11] write could not complete without blocking + # https://github.com/travis-ci/travis-ci/issues/8920 + import fcntl + + from moztreedocs.package import distribution_files + from moztreedocs.upload import s3_set_redirects, s3_upload + + fcntl.fcntl(1, fcntl.F_SETFL, 0) + + # Files are uploaded to multiple locations: + # + # <project>/latest + # <project>/<version> + # + # This allows multiple projects and versions to be stored in the + # S3 bucket. + + files = list(distribution_files(root)) + key_prefixes = [] + if version: + key_prefixes.append("%s/%s" % (project, version)) + + # Until we redirect / to main/latest, upload the main docs + # to the root. + if project == "main": + key_prefixes.append("") + + key_prefixes.append(unique_id) + + with open(os.path.join(DOC_ROOT, "config.yml"), "r") as fh: + redirects = yaml.safe_load(fh)["redirects"] + + redirects = {k.strip("/"): v.strip("/") for k, v in redirects.items()} + + all_redirects = {} + + for prefix in key_prefixes: + s3_upload(files, prefix) + + # Don't setup redirects for the "version" or "uuid" prefixes since + # we are exceeding a 50 redirect limit and external things are + # unlikely to link there anyway (see bug 1614908). + if (version and prefix.endswith(version)) or prefix == unique_id: + continue + + if prefix: + prefix += "/" + all_redirects.update({prefix + k: prefix + v for k, v in redirects.items()}) + + print("Redirects currently staged") + pprint(all_redirects, indent=1) + + s3_set_redirects(all_redirects) + + unique_link = BASE_LINK + unique_id + "/index.html" + print("Uploaded documentation can be accessed here " + unique_link) + + +@SubCommand( + "doc", + "mach-telemetry", + description="Generate documentation from Glean metrics.yaml files", +) +def generate_telemetry_docs(command_context): + args = [ + sys.executable, + "-m" "glean_parser", + "translate", + "-f", + "markdown", + "-o", + os.path.join(topsrcdir, "python/mach/docs/"), + os.path.join(topsrcdir, "python/mach/pings.yaml"), + os.path.join(topsrcdir, "python/mach/metrics.yaml"), + ] + metrics_paths = [ + handler.metrics_path + for handler in Registrar.command_handlers.values() + if handler.metrics_path is not None + ] + args.extend( + [os.path.join(command_context.topsrcdir, path) for path in set(metrics_paths)] + ) + subprocess.check_call(args) + + +@SubCommand( + "doc", + "show-targets", + description="List all reference targets. Requires the docs to have been built.", +) +@CommandArgument( + "--format", default="html", dest="fmt", help="Documentation format used." +) +@CommandArgument( + "--outdir", default=None, metavar="DESTINATION", help="Where output was written." +) +def show_reference_targets(command_context, fmt="html", outdir=None): + command_context.activate_virtualenv() + command_context.virtualenv_manager.install_pip_requirements( + os.path.join(here, "requirements.txt") + ) + + import sphinx.ext.intersphinx + + outdir = outdir or os.path.join(command_context.topobjdir, "docs") + inv_path = os.path.join(outdir, fmt, "objects.inv") + + if not os.path.exists(inv_path): + return die( + "object inventory not found: {inv_path}.\n" + "Rebuild the docs and rerun this command" + ) + sphinx.ext.intersphinx.inspect_main([inv_path]) + + +def die(msg, exit_code=1): + msg = "%s %s: %s" % (sys.argv[0], sys.argv[1], msg) + print(msg, file=sys.stderr) + return exit_code + + +def dieWithTestFailure(msg, exit_code=1): + for m in msg.split("\n"): + msg = "TEST-UNEXPECTED-FAILURE | %s %s | %s" % (sys.argv[0], sys.argv[1], m) + print(msg, file=sys.stderr) + return exit_code diff --git a/tools/moztreedocs/package.py b/tools/moztreedocs/package.py new file mode 100644 index 0000000000..b8db23ee87 --- /dev/null +++ b/tools/moztreedocs/package.py @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, # You can obtain one at http://mozilla.org/MPL/2.0/. + +import os + +from mozpack.archive import create_tar_gz_from_files +from mozpack.files import FileFinder + + +def distribution_files(root): + """Find all files suitable for distributing. + + Given the path to generated Sphinx documentation, returns an iterable + of (path, BaseFile) for files that should be archived, uploaded, etc. + Paths are relative to given root directory. + """ + finder = FileFinder(root, ignore=("_staging", "_venv")) + return finder.find("**") + + +def create_tarball(filename, root): + """Create a tar.gz archive of docs in a directory.""" + files = dict(distribution_files(root)) + + with open(filename, "wb") as fh: + create_tar_gz_from_files( + fh, files, filename=os.path.basename(filename), compresslevel=6 + ) diff --git a/tools/moztreedocs/upload.py b/tools/moztreedocs/upload.py new file mode 100644 index 0000000000..a607135174 --- /dev/null +++ b/tools/moztreedocs/upload.py @@ -0,0 +1,171 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, # You can obtain one at http://mozilla.org/MPL/2.0/. + +import io +import mimetypes +import os +import sys +from concurrent import futures +from pprint import pprint + +import boto3 +import botocore +import requests +from mozbuild.util import memoize + + +@memoize +def create_aws_session(): + """ + This function creates an aws session that is + shared between upload and delete both. + """ + region = "us-west-2" + level = os.environ.get("MOZ_SCM_LEVEL", "1") + bucket = { + "1": "gecko-docs.mozilla.org-l1", + "2": "gecko-docs.mozilla.org-l2", + "3": "gecko-docs.mozilla.org", + }[level] + secrets_url = "http://taskcluster/secrets/v1/secret/" + secrets_url += "project/releng/gecko/build/level-{}/gecko-docs-upload".format(level) + + # Get the credentials from the TC secrets service. Note that these + # differ per SCM level + if "TASK_ID" in os.environ: + print("Using AWS credentials from the secrets service") + session = requests.Session() + res = session.get(secrets_url) + res.raise_for_status() + secret = res.json()["secret"] + session = boto3.session.Session( + aws_access_key_id=secret["AWS_ACCESS_KEY_ID"], + aws_secret_access_key=secret["AWS_SECRET_ACCESS_KEY"], + region_name=region, + ) + else: + print("Trying to use your AWS credentials..") + session = boto3.session.Session(region_name=region) + + s3 = session.client("s3", config=botocore.client.Config(max_pool_connections=20)) + + return s3, bucket + + +@memoize +def get_s3_keys(s3, bucket): + kwargs = {"Bucket": bucket} + all_keys = [] + while True: + response = s3.list_objects_v2(**kwargs) + for obj in response["Contents"]: + all_keys.append(obj["Key"]) + + try: + kwargs["ContinuationToken"] = response["NextContinuationToken"] + except KeyError: + break + + return all_keys + + +def s3_set_redirects(redirects): + s3, bucket = create_aws_session() + + configuration = {"IndexDocument": {"Suffix": "index.html"}, "RoutingRules": []} + + for path, redirect in redirects.items(): + rule = { + "Condition": {"KeyPrefixEquals": path}, + "Redirect": {"ReplaceKeyPrefixWith": redirect}, + } + if os.environ.get("MOZ_SCM_LEVEL") == "3": + rule["Redirect"]["HostName"] = "firefox-source-docs.mozilla.org" + + configuration["RoutingRules"].append(rule) + + s3.put_bucket_website( + Bucket=bucket, + WebsiteConfiguration=configuration, + ) + + +def s3_delete_missing(files, key_prefix=None): + """Delete files in the S3 bucket. + + Delete files on the S3 bucket that doesn't match the files + given as the param. If the key_prefix is not specified, missing + files that has main/ as a prefix will be removed. Otherwise, it + will remove files with the same prefix as key_prefix. + """ + s3, bucket = create_aws_session() + files_on_server = get_s3_keys(s3, bucket) + if key_prefix: + files_on_server = [ + path for path in files_on_server if path.startswith(key_prefix) + ] + else: + files_on_server = [ + path for path in files_on_server if not path.startswith("main/") + ] + files = [key_prefix + "/" + path if key_prefix else path for path, f in files] + files_to_delete = [path for path in files_on_server if path not in files] + + query_size = 1000 + while files_to_delete: + keys_to_remove = [{"Key": key} for key in files_to_delete[:query_size]] + response = s3.delete_objects( + Bucket=bucket, + Delete={ + "Objects": keys_to_remove, + }, # NOQA + ) + pprint(response, indent=2) + files_to_delete = files_to_delete[query_size:] + + +def s3_upload(files, key_prefix=None): + """Upload files to an S3 bucket. + + ``files`` is an iterable of ``(path, BaseFile)`` (typically from a + mozpack Finder). + + Keys in the bucket correspond to source filenames. If ``key_prefix`` is + defined, key names will be ``<key_prefix>/<path>``. + """ + s3, bucket = create_aws_session() + + def upload(f, path, bucket, key, extra_args): + # Need to flush to avoid buffering/interleaving from multiple threads. + sys.stdout.write("uploading %s to %s\n" % (path, key)) + sys.stdout.flush() + s3.upload_fileobj(f, bucket, key, ExtraArgs=extra_args) + + fs = [] + with futures.ThreadPoolExecutor(20) as e: + for path, f in files: + content_type, content_encoding = mimetypes.guess_type(path) + extra_args = {} + if content_type: + if content_type.startswith("text/"): + content_type += '; charset="utf-8"' + extra_args["ContentType"] = content_type + if content_encoding: + extra_args["ContentEncoding"] = content_encoding + + if key_prefix: + key = "%s/%s" % (key_prefix, path) + else: + key = path + + # The file types returned by mozpack behave like file objects. But + # they don't accept an argument to read(). So we wrap in a BytesIO. + fs.append( + e.submit(upload, io.BytesIO(f.read()), path, bucket, key, extra_args) + ) + + s3_delete_missing(files, key_prefix) + # Need to do this to catch any exceptions. + for f in fs: + f.result() |