Adding upstream version 124.0.1.upstream/124.0.1

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 254 insertions, 0 deletions

diff --git a/build/pgo/profileserver.py b/build/pgo/profileserver.py
new file mode 100755
index 0000000000..6eab7889b0
--- /dev/null
+++ b/build/pgo/profileserver.py
@@ -0,0 +1,254 @@
+#!/usr/bin/python
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import glob
+import json
+import os
+import subprocess
+import sys
+
+import mozcrash
+from mozbuild.base import BinaryNotFoundException, MozbuildObject
+from mozfile import TemporaryDirectory
+from mozhttpd import MozHttpd
+from mozprofile import FirefoxProfile, Preferences
+from mozprofile.permissions import ServerLocations
+from mozrunner import CLI, FirefoxRunner
+from six import string_types
+
+PORT = 8888
+
+PATH_MAPPINGS = {
+    "/webkit/PerformanceTests": "third_party/webkit/PerformanceTests",
+    # It is tempting to map to `testing/talos/talos/tests` instead, to avoid
+    # writing `tests/` in every path, but we can't do that because some files
+    # refer to scripts located in `../..`.
+    "/talos": "testing/talos/talos",
+}
+
+
+def get_crashreports(directory, name=None):
+    rc = 0
+    upload_path = os.environ.get("UPLOAD_PATH")
+    if not upload_path:
+        upload_path = os.environ.get("UPLOAD_DIR")
+    if upload_path:
+        # For automation, log the minidumps with stackwalk and get them moved to
+        # the artifacts directory.
+        fetches_dir = os.environ.get("MOZ_FETCHES_DIR")
+        if not fetches_dir:
+            raise Exception(
+                "Unable to process minidump in automation because "
+                "$MOZ_FETCHES_DIR is not set in the environment"
+            )
+        stackwalk_binary = os.path.join(
+            fetches_dir, "minidump-stackwalk", "minidump-stackwalk"
+        )
+        if sys.platform == "win32":
+            stackwalk_binary += ".exe"
+        minidump_path = os.path.join(directory, "minidumps")
+        rc = mozcrash.check_for_crashes(
+            minidump_path,
+            symbols_path=fetches_dir,
+            stackwalk_binary=stackwalk_binary,
+            dump_save_path=upload_path,
+            test_name=name,
+        )
+    return rc
+
+
+if __name__ == "__main__":
+    cli = CLI()
+    debug_args, interactive = cli.debugger_arguments()
+    runner_args = cli.runner_args()
+
+    build = MozbuildObject.from_environment()
+
+    binary = runner_args.get("binary")
+    if not binary:
+        try:
+            binary = build.get_binary_path(where="staged-package")
+        except BinaryNotFoundException as e:
+            print("{}\n\n{}\n".format(e, e.help()))
+            sys.exit(1)
+    binary = os.path.normpath(os.path.abspath(binary))
+
+    path_mappings = {
+        k: os.path.join(build.topsrcdir, v) for k, v in PATH_MAPPINGS.items()
+    }
+    httpd = MozHttpd(
+        port=PORT,
+        docroot=os.path.join(build.topsrcdir, "build", "pgo"),
+        path_mappings=path_mappings,
+    )
+    httpd.start(block=False)
+
+    sp3_httpd = MozHttpd(
+        port=8000,
+        docroot=os.path.join(
+            build.topsrcdir, "third_party", "webkit", "PerformanceTests", "Speedometer3"
+        ),
+        path_mappings=path_mappings,
+    )
+    sp3_httpd.start(block=False)
+    print("started SP3 server on port 8000")
+    locations = ServerLocations()
+    locations.add_host(host="127.0.0.1", port=PORT, options="primary,privileged")
+
+    old_profraw_files = glob.glob("*.profraw")
+    for f in old_profraw_files:
+        os.remove(f)
+
+    with TemporaryDirectory() as profilePath:
+        # TODO: refactor this into mozprofile
+        profile_data_dir = os.path.join(build.topsrcdir, "testing", "profiles")
+        with open(os.path.join(profile_data_dir, "profiles.json"), "r") as fh:
+            base_profiles = json.load(fh)["profileserver"]
+
+        prefpaths = [
+            os.path.join(profile_data_dir, profile, "user.js")
+            for profile in base_profiles
+        ]
+
+        prefs = {}
+        for path in prefpaths:
+            prefs.update(Preferences.read_prefs(path))
+
+        interpolation = {"server": "%s:%d" % httpd.httpd.server_address}
+        sp3_interpolation = {"server": "%s:%d" % sp3_httpd.httpd.server_address}
+        for k, v in prefs.items():
+            if isinstance(v, string_types):
+                v = v.format(**interpolation)
+            prefs[k] = Preferences.cast(v)
+
+        profile = FirefoxProfile(
+            profile=profilePath,
+            preferences=prefs,
+            addons=[
+                os.path.join(
+                    build.topsrcdir, "tools", "quitter", "quitter@mozilla.org.xpi"
+                )
+            ],
+            locations=locations,
+        )
+
+        env = os.environ.copy()
+        env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
+        env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
+        env["XPCOM_DEBUG_BREAK"] = "warn"
+        # We disable sandboxing to make writing profiling data actually work
+        # Bug 1553850 considers fixing this.
+        env["MOZ_DISABLE_CONTENT_SANDBOX"] = "1"
+        env["MOZ_DISABLE_RDD_SANDBOX"] = "1"
+        env["MOZ_DISABLE_SOCKET_PROCESS_SANDBOX"] = "1"
+        env["MOZ_DISABLE_GPU_SANDBOX"] = "1"
+        env["MOZ_DISABLE_GMP_SANDBOX"] = "1"
+        env["MOZ_DISABLE_NPAPI_SANDBOX"] = "1"
+        env["MOZ_DISABLE_VR_SANDBOX"] = "1"
+
+        # Ensure different pids write to different files
+        env["LLVM_PROFILE_FILE"] = "default_%p_random_%m.profraw"
+
+        # Write to an output file if we're running in automation
+        process_args = {"universal_newlines": True}
+        if "UPLOAD_PATH" in env:
+            process_args["logfile"] = os.path.join(
+                env["UPLOAD_PATH"], "profile-run-1.log"
+            )
+
+        # Run Firefox a first time to initialize its profile
+        runner = FirefoxRunner(
+            profile=profile,
+            binary=binary,
+            cmdargs=["data:text/html,<script>Quitter.quit()</script>"],
+            env=env,
+            process_args=process_args,
+        )
+        runner.start()
+        ret = runner.wait()
+        if ret:
+            print("Firefox exited with code %d during profile initialization" % ret)
+            logfile = process_args.get("logfile")
+            if logfile:
+                print("Firefox output (%s):" % logfile)
+                with open(logfile) as f:
+                    print(f.read())
+            sp3_httpd.stop()
+            httpd.stop()
+            get_crashreports(profilePath, name="Profile initialization")
+            sys.exit(ret)
+
+        jarlog = os.getenv("JARLOG_FILE")
+        if jarlog:
+            env["MOZ_JAR_LOG_FILE"] = os.path.abspath(jarlog)
+            print("jarlog: %s" % env["MOZ_JAR_LOG_FILE"])
+            if os.path.exists(jarlog):
+                os.remove(jarlog)
+
+        if "UPLOAD_PATH" in env:
+            process_args["logfile"] = os.path.join(
+                env["UPLOAD_PATH"], "profile-run-2.log"
+            )
+        cmdargs = ["http://localhost:%d/index.html" % PORT]
+        runner = FirefoxRunner(
+            profile=profile,
+            binary=binary,
+            cmdargs=cmdargs,
+            env=env,
+            process_args=process_args,
+        )
+        runner.start(debug_args=debug_args, interactive=interactive)
+        ret = runner.wait()
+        sp3_httpd.stop()
+        httpd.stop()
+        if ret:
+            print("Firefox exited with code %d during profiling" % ret)
+            logfile = process_args.get("logfile")
+            if logfile:
+                print("Firefox output (%s):" % logfile)
+                with open(logfile) as f:
+                    print(f.read())
+            get_crashreports(profilePath, name="Profiling run")
+            sys.exit(ret)
+
+        # Try to move the crash reports to the artifacts even if Firefox appears
+        # to exit successfully, in case there's a crash that doesn't set the
+        # return code to non-zero for some reason.
+        if get_crashreports(profilePath, name="Firefox exited successfully?") != 0:
+            print("Firefox exited successfully, but produced a crashreport")
+            sys.exit(1)
+
+        llvm_profdata = env.get("LLVM_PROFDATA")
+        if llvm_profdata:
+            profraw_files = glob.glob("*.profraw")
+            if not profraw_files:
+                print(
+                    "Could not find profraw files in the current directory: %s"
+                    % os.getcwd()
+                )
+                sys.exit(1)
+
+            merged_profdata = "merged.profdata"
+            merge_cmd = [
+                llvm_profdata,
+                "merge",
+                "-o",
+                merged_profdata,
+            ] + profraw_files
+            rc = subprocess.call(merge_cmd)
+            if rc != 0:
+                print("INFRA-ERROR: Failed to merge profile data. Corrupt profile?")
+                # exit with TBPL_RETRY
+                sys.exit(4)
+
+            # llvm-profdata may fail while still exiting without an error.
+            if not os.path.isfile(merged_profdata):
+                print(merged_profdata, "was not created", file=sys.stderr)
+                sys.exit(1)
+
+            if os.path.getsize(merged_profdata) == 0:
+                print(merged_profdata, "was created but it is empty", file=sys.stderr)
+                sys.exit(1)