Adding upstream version 115.7.0esr.upstream/115.7.0esrupstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 689 insertions, 0 deletions

diff --git a/js/src/devtools/rootAnalysis/mach_commands.py b/js/src/devtools/rootAnalysis/mach_commands.py
new file mode 100644
index 0000000000..f21cbaf45c
--- /dev/null
+++ b/js/src/devtools/rootAnalysis/mach_commands.py
@@ -0,0 +1,689 @@
+# -*- coding: utf-8 -*-
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import argparse
+import html
+import json
+import logging
+import os
+import re
+import textwrap
+import webbrowser
+
+# Command files like this are listed in build/mach_initialize.py in alphabetical
+# order, but we need to access commands earlier in the sorted order to grab
+# their arguments. Force them to load now.
+import mozbuild.artifact_commands  # NOQA: F401
+import mozbuild.build_commands  # NOQA: F401
+import mozhttpd
+from mach.base import FailedCommandError, MachError
+from mach.decorators import Command, CommandArgument, SubCommand
+from mach.registrar import Registrar
+from mozbuild.mozconfig import MozconfigLoader
+
+
+# Use a decorator to copy command arguments off of the named command. Instead
+# of a decorator, this could be straight code that edits eg
+# MachCommands.build_shell._mach_command.arguments, but that looked uglier.
+def inherit_command_args(command, subcommand=None):
+    """Decorator for inheriting all command-line arguments from `mach build`.
+
+    This should come earlier in the source file than @Command or @SubCommand,
+    because it relies on that decorator having run first."""
+
+    def inherited(func):
+        handler = Registrar.command_handlers.get(command)
+        if handler is not None and subcommand is not None:
+            handler = handler.subcommand_handlers.get(subcommand)
+        if handler is None:
+            raise MachError(
+                "{} command unknown or not yet loaded".format(
+                    command if subcommand is None else command + " " + subcommand
+                )
+            )
+        func._mach_command.arguments.extend(handler.arguments)
+        return func
+
+    return inherited
+
+
+def state_dir():
+    return os.environ.get("MOZBUILD_STATE_PATH", os.path.expanduser("~/.mozbuild"))
+
+
+def tools_dir():
+    if os.environ.get("MOZ_FETCHES_DIR"):
+        # In automation, tools are provided by toolchain dependencies.
+        return os.path.join(os.environ["HOME"], os.environ["MOZ_FETCHES_DIR"])
+
+    # In development, `mach hazard bootstrap` installs the tools separately
+    # to avoid colliding with the "main" compiler versions, which can
+    # change separately (and the precompiled sixgill and compiler version
+    # must match exactly).
+    return os.path.join(state_dir(), "hazard-tools")
+
+
+def sixgill_dir():
+    return os.path.join(tools_dir(), "sixgill")
+
+
+def gcc_dir():
+    return os.path.join(tools_dir(), "gcc")
+
+
+def script_dir(command_context):
+    return os.path.join(command_context.topsrcdir, "js/src/devtools/rootAnalysis")
+
+
+def get_work_dir(command_context, project, given):
+    if given is not None:
+        return given
+    return os.path.join(command_context.topsrcdir, "haz-" + project)
+
+
+def get_objdir(command_context, kwargs):
+    project = kwargs["project"]
+    objdir = kwargs["haz_objdir"]
+    if objdir is None:
+        objdir = os.environ.get("HAZ_OBJDIR")
+    if objdir is None:
+        objdir = os.path.join(command_context.topsrcdir, "obj-analyzed-" + project)
+    return objdir
+
+
+def ensure_dir_exists(dir):
+    os.makedirs(dir, exist_ok=True)
+    return dir
+
+
+# Force the use of hazard-compatible installs of tools.
+def setup_env_for_tools(env):
+    gccbin = os.path.join(gcc_dir(), "bin")
+    env["CC"] = os.path.join(gccbin, "gcc")
+    env["CXX"] = os.path.join(gccbin, "g++")
+    env["PATH"] = "{sixgill_dir}/usr/bin:{gccbin}:{PATH}".format(
+        sixgill_dir=sixgill_dir(), gccbin=gccbin, PATH=env["PATH"]
+    )
+
+
+def setup_env_for_shell(env, shell):
+    """Add JS shell directory to dynamic lib search path"""
+    for var in ("LD_LIBRARY_PATH", "DYLD_LIBRARY_PATH"):
+        env[var] = ":".join(p for p in (env.get(var), os.path.dirname(shell)) if p)
+
+
+@Command(
+    "hazards",
+    category="build",
+    order="declaration",
+    description="Commands for running the static analysis for GC rooting hazards",
+)
+def hazards(command_context):
+    """Commands related to performing the GC rooting hazard analysis"""
+    print("See `mach hazards --help` for a list of subcommands")
+
+
+@inherit_command_args("artifact", "toolchain")
+@SubCommand(
+    "hazards",
+    "bootstrap",
+    description="Install prerequisites for the hazard analysis",
+)
+def bootstrap(command_context, **kwargs):
+    orig_dir = os.getcwd()
+    os.chdir(ensure_dir_exists(tools_dir()))
+    try:
+        kwargs["from_build"] = ("linux64-gcc-sixgill", "linux64-gcc-9")
+        command_context._mach_context.commands.dispatch(
+            "artifact", command_context._mach_context, subcommand="toolchain", **kwargs
+        )
+    finally:
+        os.chdir(orig_dir)
+
+
+CLOBBER_CHOICES = {"objdir", "work", "shell", "all"}
+
+
+@SubCommand("hazards", "clobber", description="Clean up hazard-related files")
+@CommandArgument("--project", default="browser", help="Build the given project.")
+@CommandArgument("--application", dest="project", help="Build the given project.")
+@CommandArgument("--haz-objdir", default=None, help="Hazard analysis objdir.")
+@CommandArgument(
+    "--work-dir", default=None, help="Directory for output and working files."
+)
+@CommandArgument(
+    "what",
+    default=["objdir", "work"],
+    nargs="*",
+    help="Target to clobber, must be one of {{{}}} (default "
+    "objdir and work).".format(", ".join(CLOBBER_CHOICES)),
+)
+def clobber(command_context, what, **kwargs):
+    from mozbuild.controller.clobber import Clobberer
+
+    what = set(what)
+    if "all" in what:
+        what.update(CLOBBER_CHOICES)
+    invalid = what - CLOBBER_CHOICES
+    if invalid:
+        print(
+            "Unknown clobber target(s): {}. Choose from {{{}}}".format(
+                ", ".join(invalid), ", ".join(CLOBBER_CHOICES)
+            )
+        )
+        return 1
+
+    try:
+        substs = command_context.substs
+    except BuildEnvironmentNotFoundException:
+        substs = {}
+
+    if "objdir" in what:
+        objdir = get_objdir(command_context, kwargs)
+        print(f"removing {objdir}")
+        Clobberer(command_context.topsrcdir, objdir, substs).remove_objdir(full=True)
+    if "work" in what:
+        project = kwargs["project"]
+        work_dir = get_work_dir(command_context, project, kwargs["work_dir"])
+        print(f"removing {work_dir}")
+        Clobberer(command_context.topsrcdir, work_dir, substs).remove_objdir(full=True)
+    if "shell" in what:
+        objdir = os.path.join(command_context.topsrcdir, "obj-haz-shell")
+        print(f"removing {objdir}")
+        Clobberer(command_context.topsrcdir, objdir, substs).remove_objdir(full=True)
+
+
+@inherit_command_args("build")
+@SubCommand(
+    "hazards", "build-shell", description="Build a shell for the hazard analysis"
+)
+@CommandArgument(
+    "--mozconfig",
+    default=None,
+    metavar="FILENAME",
+    help="Build with the given mozconfig.",
+)
+def build_shell(command_context, **kwargs):
+    """Build a JS shell to use to run the rooting hazard analysis."""
+    # The JS shell requires some specific configuration settings to execute
+    # the hazard analysis code, and configuration is done via mozconfig.
+    # Subprocesses find MOZCONFIG in the environment, so we can't just
+    # modify the settings in this process's loaded version. Pass it through
+    # the environment.
+
+    default_mozconfig = "js/src/devtools/rootAnalysis/mozconfig.haz_shell"
+    mozconfig_path = (
+        kwargs.pop("mozconfig", None)
+        or os.environ.get("MOZCONFIG")
+        or default_mozconfig
+    )
+    mozconfig_path = os.path.join(command_context.topsrcdir, mozconfig_path)
+    loader = MozconfigLoader(command_context.topsrcdir)
+    mozconfig = loader.read_mozconfig(mozconfig_path)
+
+    # Validate the mozconfig settings in case the user overrode the default.
+    configure_args = mozconfig["configure_args"]
+    if "--enable-ctypes" not in configure_args:
+        raise FailedCommandError(
+            "ctypes required in hazard JS shell, mozconfig=" + mozconfig_path
+        )
+
+    # Transmit the mozconfig location to build subprocesses.
+    os.environ["MOZCONFIG"] = mozconfig_path
+
+    setup_env_for_tools(os.environ)
+
+    # Set a default objdir for the shell, for developer builds.
+    os.environ.setdefault(
+        "MOZ_OBJDIR", os.path.join(command_context.topsrcdir, "obj-haz-shell")
+    )
+
+    return command_context._mach_context.commands.dispatch(
+        "build", command_context._mach_context, **kwargs
+    )
+
+
+def read_json_file(filename):
+    with open(filename) as fh:
+        return json.load(fh)
+
+
+def ensure_shell(command_context, objdir):
+    if objdir is None:
+        objdir = os.path.join(command_context.topsrcdir, "obj-haz-shell")
+
+    try:
+        binaries = read_json_file(os.path.join(objdir, "binaries.json"))
+        info = [b for b in binaries["programs"] if b["program"] == "js"][0]
+        return os.path.join(objdir, info["install_target"], "js")
+    except (OSError, KeyError):
+        raise FailedCommandError(
+            """\
+no shell found in %s -- must build the JS shell with `mach hazards build-shell` first"""
+            % objdir
+        )
+
+
+def validate_mozconfig(command_context, kwargs):
+    app = kwargs.pop("project")
+    default_mozconfig = "js/src/devtools/rootAnalysis/mozconfig.%s" % app
+    mozconfig_path = (
+        kwargs.pop("mozconfig", None)
+        or os.environ.get("MOZCONFIG")
+        or default_mozconfig
+    )
+    mozconfig_path = os.path.join(command_context.topsrcdir, mozconfig_path)
+
+    loader = MozconfigLoader(command_context.topsrcdir)
+    mozconfig = loader.read_mozconfig(mozconfig_path)
+    configure_args = mozconfig["configure_args"]
+
+    # Require an explicit --enable-project/application=APP (even if you just
+    # want to build the default browser project.)
+    if (
+        "--enable-project=%s" % app not in configure_args
+        and "--enable-application=%s" % app not in configure_args
+    ):
+        raise FailedCommandError(
+            textwrap.dedent(
+                f"""\
+            mozconfig {mozconfig_path} builds wrong project.
+            unset MOZCONFIG to use the default {default_mozconfig}\
+            """
+            )
+        )
+
+    if not any("--with-compiler-wrapper" in a for a in configure_args):
+        raise FailedCommandError(
+            "mozconfig must wrap compiles with --with-compiler-wrapper"
+        )
+
+    return mozconfig_path
+
+
+@inherit_command_args("build")
+@SubCommand(
+    "hazards",
+    "gather",
+    description="Gather analysis data by compiling the given project",
+)
+@CommandArgument("--project", default="browser", help="Build the given project.")
+@CommandArgument("--application", dest="project", help="Build the given project.")
+@CommandArgument(
+    "--haz-objdir", default=None, help="Write object files to this directory."
+)
+@CommandArgument(
+    "--work-dir", default=None, help="Directory for output and working files."
+)
+def gather_hazard_data(command_context, **kwargs):
+    """Gather analysis information by compiling the tree"""
+    project = kwargs["project"]
+    objdir = get_objdir(command_context, kwargs)
+
+    work_dir = get_work_dir(command_context, project, kwargs["work_dir"])
+    ensure_dir_exists(work_dir)
+    with open(os.path.join(work_dir, "defaults.py"), "wt") as fh:
+        data = textwrap.dedent(
+            """\
+            analysis_scriptdir = "{script_dir}"
+            objdir = "{objdir}"
+            source = "{srcdir}"
+            sixgill = "{sixgill_dir}/usr/libexec/sixgill"
+            sixgill_bin = "{sixgill_dir}/usr/bin"
+        """
+        ).format(
+            script_dir=script_dir(command_context),
+            objdir=objdir,
+            srcdir=command_context.topsrcdir,
+            sixgill_dir=sixgill_dir(),
+            gcc_dir=gcc_dir(),
+        )
+        fh.write(data)
+
+    buildscript = " ".join(
+        [
+            command_context.topsrcdir + "/mach hazards compile",
+            *kwargs.get("what", []),
+            "--job-size=3.0",  # Conservatively estimate 3GB/process
+            "--project=" + project,
+            "--haz-objdir=" + objdir,
+        ]
+    )
+    args = [
+        os.path.join(script_dir(command_context), "run_complete"),
+        "--foreground",
+        "--no-logs",
+        "--build-root=" + objdir,
+        "--wrap-dir=" + sixgill_dir() + "/usr/libexec/sixgill/scripts/wrap_gcc",
+        "--work-dir=work",
+        "-b",
+        sixgill_dir() + "/usr/bin",
+        "--buildcommand=" + buildscript,
+        ".",
+    ]
+
+    return command_context.run_process(args=args, cwd=work_dir, pass_thru=True)
+
+
+@inherit_command_args("build")
+@SubCommand("hazards", "compile", description=argparse.SUPPRESS)
+@CommandArgument(
+    "--mozconfig",
+    default=None,
+    metavar="FILENAME",
+    help="Build with the given mozconfig.",
+)
+@CommandArgument("--project", default="browser", help="Build the given project.")
+@CommandArgument("--application", dest="project", help="Build the given project.")
+@CommandArgument(
+    "--haz-objdir",
+    default=os.environ.get("HAZ_OBJDIR"),
+    help="Write object files to this directory.",
+)
+def inner_compile(command_context, **kwargs):
+    """Build a source tree and gather analysis information while running
+    under the influence of the analysis collection server."""
+
+    env = os.environ
+
+    # Check whether we are running underneath the manager (and therefore
+    # have a server to talk to).
+    if "XGILL_CONFIG" not in env:
+        raise FailedCommandError(
+            "no sixgill manager detected. `mach hazards compile` "
+            + "should only be run from `mach hazards gather`"
+        )
+
+    mozconfig_path = validate_mozconfig(command_context, kwargs)
+
+    # Communicate mozconfig to build subprocesses.
+    env["MOZCONFIG"] = os.path.join(command_context.topsrcdir, mozconfig_path)
+
+    # hazard mozconfigs need to find binaries in .mozbuild
+    env["MOZBUILD_STATE_PATH"] = state_dir()
+
+    # Suppress the gathering of sources, to save disk space and memory.
+    env["XGILL_NO_SOURCE"] = "1"
+
+    setup_env_for_tools(env)
+
+    if "haz_objdir" in kwargs:
+        env["MOZ_OBJDIR"] = kwargs.pop("haz_objdir")
+
+    return command_context._mach_context.commands.dispatch(
+        "build", command_context._mach_context, **kwargs
+    )
+
+
+@SubCommand(
+    "hazards", "analyze", description="Analyzed gathered data for rooting hazards"
+)
+@CommandArgument(
+    "--project",
+    default="browser",
+    help="Analyze the output for the given project.",
+)
+@CommandArgument("--application", dest="project", help="Build the given project.")
+@CommandArgument(
+    "--shell-objdir",
+    default=None,
+    help="objdir containing the optimized JS shell for running the analysis.",
+)
+@CommandArgument(
+    "--work-dir", default=None, help="Directory for output and working files."
+)
+@CommandArgument(
+    "--jobs", "-j", default=None, type=int, help="Number of parallel analyzers."
+)
+@CommandArgument(
+    "--verbose",
+    "-v",
+    default=False,
+    action="store_true",
+    help="Display executed commands.",
+)
+@CommandArgument(
+    "--from-stage",
+    default=None,
+    help="Stage to begin running at ('list' to see all).",
+)
+@CommandArgument(
+    "extra",
+    nargs=argparse.REMAINDER,
+    default=(),
+    help="Remaining non-optional arguments to analyze.py script",
+)
+def analyze(
+    command_context,
+    project,
+    shell_objdir,
+    work_dir,
+    jobs,
+    verbose,
+    from_stage,
+    extra,
+):
+    """Analyzed gathered data for rooting hazards"""
+
+    shell = ensure_shell(command_context, shell_objdir)
+    args = [
+        os.path.join(script_dir(command_context), "analyze.py"),
+        "--js",
+        shell,
+        *extra,
+    ]
+
+    if from_stage is None:
+        pass
+    elif from_stage == "list":
+        args.append("--list")
+    else:
+        args.extend(["--first", from_stage])
+
+    if jobs is not None:
+        args.extend(["-j", jobs])
+
+    if verbose:
+        args.append("-v")
+
+    setup_env_for_tools(os.environ)
+    setup_env_for_shell(os.environ, shell)
+
+    work_dir = get_work_dir(command_context, project, work_dir)
+    return command_context.run_process(args=args, cwd=work_dir, pass_thru=True)
+
+
+@SubCommand(
+    "hazards",
+    "self-test",
+    description="Run a self-test to verify hazards are detected",
+)
+@CommandArgument(
+    "--shell-objdir",
+    default=None,
+    help="objdir containing the optimized JS shell for running the analysis.",
+)
+@CommandArgument(
+    "extra",
+    nargs=argparse.REMAINDER,
+    help="Remaining non-optional arguments to pass to run-test.py",
+)
+def self_test(command_context, shell_objdir, extra):
+    """Analyzed gathered data for rooting hazards"""
+    shell = ensure_shell(command_context, shell_objdir)
+    args = [
+        os.path.join(script_dir(command_context), "run-test.py"),
+        "-v",
+        "--js",
+        shell,
+        "--sixgill",
+        os.path.join(tools_dir(), "sixgill"),
+        "--gccdir",
+        gcc_dir(),
+    ]
+    args.extend(extra)
+
+    setup_env_for_tools(os.environ)
+    setup_env_for_shell(os.environ, shell)
+
+    return command_context.run_process(args=args, pass_thru=True)
+
+
+def annotated_source(filename, query):
+    """The index page has URLs of the format <http://.../path/to/source.cpp?L=m-n#m>.
+    The `#m` part will be stripped off and used by the browser to jump to the correct line.
+    The `?L=m-n` or `?L=m` parameter will be processed here on the server to highlight
+    the given line range."""
+    linequery = query.replace("L=", "")
+    if "-" in linequery:
+        line0, line1 = linequery.split("-", 1)
+    else:
+        line0, line1 = linequery or "0", linequery or "0"
+    line0 = int(line0)
+    line1 = int(line1)
+
+    fh = open(filename, "rt")
+
+    out = "<pre>"
+    for lineno, line in enumerate(fh, 1):
+        processed = f"{lineno} <span id='{lineno}'"
+        if line0 <= lineno and lineno <= line1:
+            processed += " style='background: yellow'"
+        processed += ">" + html.escape(line.rstrip()) + "</span>\n"
+        out += processed
+
+    return out
+
+
+@SubCommand(
+    "hazards", "view", description="Display a web page describing any hazards found"
+)
+@CommandArgument(
+    "--project",
+    default="browser",
+    help="Analyze the output for the given project.",
+)
+@CommandArgument("--application", dest="project", help="Build the given project.")
+@CommandArgument(
+    "--haz-objdir", default=None, help="Write object files to this directory."
+)
+@CommandArgument(
+    "--work-dir", default=None, help="Directory for output and working files."
+)
+@CommandArgument("--port", default=6006, help="Port of the web server")
+@CommandArgument(
+    "--serve-only",
+    default=False,
+    action="store_true",
+    help="Serve only, do not navigate to page",
+)
+def view_hazards(command_context, project, haz_objdir, work_dir, port, serve_only):
+    work_dir = get_work_dir(command_context, project, work_dir)
+    haztop = os.path.basename(work_dir)
+    if haz_objdir is None:
+        haz_objdir = os.environ.get("HAZ_OBJDIR")
+    if haz_objdir is None:
+        haz_objdir = os.path.join(command_context.topsrcdir, "obj-analyzed-" + project)
+
+    httpd = None
+
+    def serve_source_file(request, path):
+        info = {"req": path}
+
+        def log(fmt, level=logging.INFO):
+            return command_context.log(level, "view-hazards", info, fmt)
+
+        if path in ("", f"{haztop}"):
+            info["dest"] = f"/{haztop}/hazards.html"
+            info["code"] = 301
+            log("serve '{req}' -> {code} {dest}")
+            return (info["code"], {"Location": info["dest"]}, "")
+
+        # Allow files to be served from the source directory or the objdir.
+        roots = (command_context.topsrcdir, haz_objdir)
+
+        try:
+            # Validate the path. Some source files have weird characters in their paths (eg "+"), but they
+            # all start with an alphanumeric or underscore.
+            command_context.log(
+                logging.DEBUG, "view-hazards", {"path": path}, "Raw path: {path}"
+            )
+            path_component = r"\w[\w\-\.\+]*"
+            if not re.match(f"({path_component}/)*{path_component}$", path):
+                raise ValueError("invalid path")
+
+            # Resolve the path to under one of the roots, and
+            # ensure that the actual file really is underneath a root directory.
+            for rootdir in roots:
+                fullpath = os.path.join(rootdir, path)
+                info["path"] = fullpath
+                fullpath = os.path.realpath(fullpath)
+                if os.path.isfile(fullpath):
+                    # symlinks between roots are ok, but not symlinks outside of the roots.
+                    tops = [
+                        d
+                        for d in roots
+                        if fullpath.startswith(os.path.realpath(d) + "/")
+                    ]
+                    if len(tops) > 0:
+                        break  # Found a file underneath a root.
+            else:
+                raise IOError("not found")
+
+            html = annotated_source(fullpath, request.query)
+            log("serve '{req}' -> 200 {path}")
+            return (
+                200,
+                {"Content-type": "text/html", "Content-length": len(html)},
+                html,
+            )
+        except (IOError, ValueError):
+            log("serve '{req}' -> 404 {path}", logging.ERROR)
+            return (
+                404,
+                {"Content-type": "text/plain"},
+                "We don't have that around here. Don't be asking for it.",
+            )
+
+    httpd = mozhttpd.MozHttpd(
+        port=port,
+        docroot=None,
+        path_mappings={"/" + haztop: work_dir},
+        urlhandlers=[
+            # Treat everything not starting with /haz-browser/ (or /haz-js/)
+            # as a source file to be processed. Everything else is served
+            # as a plain file.
+            {
+                "method": "GET",
+                "path": "/(?!haz-" + project + "/)(.*)",
+                "function": serve_source_file,
+            },
+        ],
+        log_requests=True,
+    )
+
+    # The mozhttpd request handler class eats log messages.
+    httpd.handler_class.log_message = lambda self, format, *args: command_context.log(
+        logging.INFO, "view-hazards", {}, format % args
+    )
+
+    print("Serving at %s:%s" % (httpd.host, httpd.port))
+
+    httpd.start(block=False)
+    url = httpd.get_url(f"/{haztop}/hazards.html")
+    display_url = True
+    if not serve_only:
+        try:
+            webbrowser.get().open_new_tab(url)
+            display_url = False
+        except Exception:
+            pass
+    if display_url:
+        print("Please open %s in a browser." % url)
+
+    print("Hit CTRL+c to stop server.")
+    httpd.server.join()