summaryrefslogtreecommitdiffstats
path: root/build/pgo/profileserver.py
diff options
context:
space:
mode:
Diffstat (limited to 'build/pgo/profileserver.py')
-rwxr-xr-xbuild/pgo/profileserver.py254
1 files changed, 254 insertions, 0 deletions
diff --git a/build/pgo/profileserver.py b/build/pgo/profileserver.py
new file mode 100755
index 0000000000..6eab7889b0
--- /dev/null
+++ b/build/pgo/profileserver.py
@@ -0,0 +1,254 @@
+#!/usr/bin/python
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import glob
+import json
+import os
+import subprocess
+import sys
+
+import mozcrash
+from mozbuild.base import BinaryNotFoundException, MozbuildObject
+from mozfile import TemporaryDirectory
+from mozhttpd import MozHttpd
+from mozprofile import FirefoxProfile, Preferences
+from mozprofile.permissions import ServerLocations
+from mozrunner import CLI, FirefoxRunner
+from six import string_types
+
+PORT = 8888
+
+PATH_MAPPINGS = {
+ "/webkit/PerformanceTests": "third_party/webkit/PerformanceTests",
+ # It is tempting to map to `testing/talos/talos/tests` instead, to avoid
+ # writing `tests/` in every path, but we can't do that because some files
+ # refer to scripts located in `../..`.
+ "/talos": "testing/talos/talos",
+}
+
+
+def get_crashreports(directory, name=None):
+ rc = 0
+ upload_path = os.environ.get("UPLOAD_PATH")
+ if not upload_path:
+ upload_path = os.environ.get("UPLOAD_DIR")
+ if upload_path:
+ # For automation, log the minidumps with stackwalk and get them moved to
+ # the artifacts directory.
+ fetches_dir = os.environ.get("MOZ_FETCHES_DIR")
+ if not fetches_dir:
+ raise Exception(
+ "Unable to process minidump in automation because "
+ "$MOZ_FETCHES_DIR is not set in the environment"
+ )
+ stackwalk_binary = os.path.join(
+ fetches_dir, "minidump-stackwalk", "minidump-stackwalk"
+ )
+ if sys.platform == "win32":
+ stackwalk_binary += ".exe"
+ minidump_path = os.path.join(directory, "minidumps")
+ rc = mozcrash.check_for_crashes(
+ minidump_path,
+ symbols_path=fetches_dir,
+ stackwalk_binary=stackwalk_binary,
+ dump_save_path=upload_path,
+ test_name=name,
+ )
+ return rc
+
+
+if __name__ == "__main__":
+ cli = CLI()
+ debug_args, interactive = cli.debugger_arguments()
+ runner_args = cli.runner_args()
+
+ build = MozbuildObject.from_environment()
+
+ binary = runner_args.get("binary")
+ if not binary:
+ try:
+ binary = build.get_binary_path(where="staged-package")
+ except BinaryNotFoundException as e:
+ print("{}\n\n{}\n".format(e, e.help()))
+ sys.exit(1)
+ binary = os.path.normpath(os.path.abspath(binary))
+
+ path_mappings = {
+ k: os.path.join(build.topsrcdir, v) for k, v in PATH_MAPPINGS.items()
+ }
+ httpd = MozHttpd(
+ port=PORT,
+ docroot=os.path.join(build.topsrcdir, "build", "pgo"),
+ path_mappings=path_mappings,
+ )
+ httpd.start(block=False)
+
+ sp3_httpd = MozHttpd(
+ port=8000,
+ docroot=os.path.join(
+ build.topsrcdir, "third_party", "webkit", "PerformanceTests", "Speedometer3"
+ ),
+ path_mappings=path_mappings,
+ )
+ sp3_httpd.start(block=False)
+ print("started SP3 server on port 8000")
+ locations = ServerLocations()
+ locations.add_host(host="127.0.0.1", port=PORT, options="primary,privileged")
+
+ old_profraw_files = glob.glob("*.profraw")
+ for f in old_profraw_files:
+ os.remove(f)
+
+ with TemporaryDirectory() as profilePath:
+ # TODO: refactor this into mozprofile
+ profile_data_dir = os.path.join(build.topsrcdir, "testing", "profiles")
+ with open(os.path.join(profile_data_dir, "profiles.json"), "r") as fh:
+ base_profiles = json.load(fh)["profileserver"]
+
+ prefpaths = [
+ os.path.join(profile_data_dir, profile, "user.js")
+ for profile in base_profiles
+ ]
+
+ prefs = {}
+ for path in prefpaths:
+ prefs.update(Preferences.read_prefs(path))
+
+ interpolation = {"server": "%s:%d" % httpd.httpd.server_address}
+ sp3_interpolation = {"server": "%s:%d" % sp3_httpd.httpd.server_address}
+ for k, v in prefs.items():
+ if isinstance(v, string_types):
+ v = v.format(**interpolation)
+ prefs[k] = Preferences.cast(v)
+
+ profile = FirefoxProfile(
+ profile=profilePath,
+ preferences=prefs,
+ addons=[
+ os.path.join(
+ build.topsrcdir, "tools", "quitter", "quitter@mozilla.org.xpi"
+ )
+ ],
+ locations=locations,
+ )
+
+ env = os.environ.copy()
+ env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
+ env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
+ env["XPCOM_DEBUG_BREAK"] = "warn"
+ # We disable sandboxing to make writing profiling data actually work
+ # Bug 1553850 considers fixing this.
+ env["MOZ_DISABLE_CONTENT_SANDBOX"] = "1"
+ env["MOZ_DISABLE_RDD_SANDBOX"] = "1"
+ env["MOZ_DISABLE_SOCKET_PROCESS_SANDBOX"] = "1"
+ env["MOZ_DISABLE_GPU_SANDBOX"] = "1"
+ env["MOZ_DISABLE_GMP_SANDBOX"] = "1"
+ env["MOZ_DISABLE_NPAPI_SANDBOX"] = "1"
+ env["MOZ_DISABLE_VR_SANDBOX"] = "1"
+
+ # Ensure different pids write to different files
+ env["LLVM_PROFILE_FILE"] = "default_%p_random_%m.profraw"
+
+ # Write to an output file if we're running in automation
+ process_args = {"universal_newlines": True}
+ if "UPLOAD_PATH" in env:
+ process_args["logfile"] = os.path.join(
+ env["UPLOAD_PATH"], "profile-run-1.log"
+ )
+
+ # Run Firefox a first time to initialize its profile
+ runner = FirefoxRunner(
+ profile=profile,
+ binary=binary,
+ cmdargs=["data:text/html,<script>Quitter.quit()</script>"],
+ env=env,
+ process_args=process_args,
+ )
+ runner.start()
+ ret = runner.wait()
+ if ret:
+ print("Firefox exited with code %d during profile initialization" % ret)
+ logfile = process_args.get("logfile")
+ if logfile:
+ print("Firefox output (%s):" % logfile)
+ with open(logfile) as f:
+ print(f.read())
+ sp3_httpd.stop()
+ httpd.stop()
+ get_crashreports(profilePath, name="Profile initialization")
+ sys.exit(ret)
+
+ jarlog = os.getenv("JARLOG_FILE")
+ if jarlog:
+ env["MOZ_JAR_LOG_FILE"] = os.path.abspath(jarlog)
+ print("jarlog: %s" % env["MOZ_JAR_LOG_FILE"])
+ if os.path.exists(jarlog):
+ os.remove(jarlog)
+
+ if "UPLOAD_PATH" in env:
+ process_args["logfile"] = os.path.join(
+ env["UPLOAD_PATH"], "profile-run-2.log"
+ )
+ cmdargs = ["http://localhost:%d/index.html" % PORT]
+ runner = FirefoxRunner(
+ profile=profile,
+ binary=binary,
+ cmdargs=cmdargs,
+ env=env,
+ process_args=process_args,
+ )
+ runner.start(debug_args=debug_args, interactive=interactive)
+ ret = runner.wait()
+ sp3_httpd.stop()
+ httpd.stop()
+ if ret:
+ print("Firefox exited with code %d during profiling" % ret)
+ logfile = process_args.get("logfile")
+ if logfile:
+ print("Firefox output (%s):" % logfile)
+ with open(logfile) as f:
+ print(f.read())
+ get_crashreports(profilePath, name="Profiling run")
+ sys.exit(ret)
+
+ # Try to move the crash reports to the artifacts even if Firefox appears
+ # to exit successfully, in case there's a crash that doesn't set the
+ # return code to non-zero for some reason.
+ if get_crashreports(profilePath, name="Firefox exited successfully?") != 0:
+ print("Firefox exited successfully, but produced a crashreport")
+ sys.exit(1)
+
+ llvm_profdata = env.get("LLVM_PROFDATA")
+ if llvm_profdata:
+ profraw_files = glob.glob("*.profraw")
+ if not profraw_files:
+ print(
+ "Could not find profraw files in the current directory: %s"
+ % os.getcwd()
+ )
+ sys.exit(1)
+
+ merged_profdata = "merged.profdata"
+ merge_cmd = [
+ llvm_profdata,
+ "merge",
+ "-o",
+ merged_profdata,
+ ] + profraw_files
+ rc = subprocess.call(merge_cmd)
+ if rc != 0:
+ print("INFRA-ERROR: Failed to merge profile data. Corrupt profile?")
+ # exit with TBPL_RETRY
+ sys.exit(4)
+
+ # llvm-profdata may fail while still exiting without an error.
+ if not os.path.isfile(merged_profdata):
+ print(merged_profdata, "was not created", file=sys.stderr)
+ sys.exit(1)
+
+ if os.path.getsize(merged_profdata) == 0:
+ print(merged_profdata, "was created but it is empty", file=sys.stderr)
+ sys.exit(1)