diff options
Diffstat (limited to 'testing/condprofile/condprof/creator.py')
| -rw-r--r-- | testing/condprofile/condprof/creator.py | 226 |
1 files changed, 226 insertions, 0 deletions
diff --git a/testing/condprofile/condprof/creator.py b/testing/condprofile/condprof/creator.py new file mode 100644 index 0000000000..7e03f6a8da --- /dev/null +++ b/testing/condprofile/condprof/creator.py @@ -0,0 +1,226 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" Creates or updates profiles. + +The profile creation works as following: + +For each scenario: + +- The latest indexed profile is picked on TC, if none we create a fresh profile +- The scenario is done against it +- The profile is uploaded on TC, replacing the previous one as the freshest + +For each platform we keep a changelog file that keep track of each update +with the Task ID. That offers us the ability to get a profile from a specific +date in the past. + +Artifacts are staying in TaskCluster for 3 months, and then they are removed, +so the oldest profile we can get is 3 months old. Profiles are being updated +continuously, so even after 3 months they are still getting "older". + +When Firefox changes its version, profiles from the previous version +should work as expected. Each profile tarball comes with a metadata file +that keep track of the Firefox version that was used and the profile age. +""" +import os +import tempfile +import shutil + +from arsenic import get_session +from arsenic.browsers import Firefox + +from condprof.util import fresh_profile, logger, obfuscate_file, obfuscate, get_version +from condprof.helpers import close_extra_windows +from condprof.scenarii import scenarii +from condprof.client import get_profile, ProfileNotFoundError +from condprof.archiver import Archiver +from condprof.customization import get_customization +from condprof.metadata import Metadata + + +START, INIT_GECKODRIVER, START_SESSION, START_SCENARIO = range(4) + + +class ProfileCreator: + def __init__( + self, + scenario, + customization, + archive, + changelog, + force_new, + env, + skip_logs=False, + remote_test_root="/sdcard/test_root/", + ): + self.env = env + self.scenario = scenario + self.customization = customization + self.archive = archive + self.changelog = changelog + self.force_new = force_new + self.skip_logs = skip_logs + self.remote_test_root = remote_test_root + self.customization_data = get_customization(customization) + self.tmp_dir = None + + # Make a temporary directory for the logs if an + # archive dir is not provided + if not self.archive: + self.tmp_dir = tempfile.mkdtemp() + + def _log_filename(self, name): + filename = "%s-%s-%s.log" % ( + name, + self.scenario, + self.customization_data["name"], + ) + return os.path.join(self.archive or self.tmp_dir, filename) + + async def run(self, headless=True): + logger.info( + "Building %s x %s" % (self.scenario, self.customization_data["name"]) + ) + + if self.scenario in self.customization_data.get("ignore_scenario", []): + logger.info("Skipping (ignored scenario in that customization)") + return + + filter_by_platform = self.customization_data.get("platforms") + if filter_by_platform and self.env.target_platform not in filter_by_platform: + logger.info("Skipping (ignored platform in that customization)") + return + + with self.env.get_device( + 2828, verbose=True, remote_test_root=self.remote_test_root + ) as device: + try: + with self.env.get_browser(): + metadata = await self.build_profile(device, headless) + except Exception: + raise + finally: + if not self.skip_logs: + self.env.dump_logs() + + if not self.archive: + return + + logger.info("Creating generic archive") + names = ["profile-%(platform)s-%(name)s-%(customization)s.tgz"] + if metadata["name"] == "full" and metadata["customization"] == "default": + names = [ + "profile-%(platform)s-%(name)s-%(customization)s.tgz", + "profile-v%(version)s-%(platform)s-%(name)s-%(customization)s.tgz", + ] + + for name in names: + # remove `cache` from profile + shutil.rmtree(os.path.join(self.env.profile, "cache"), ignore_errors=True) + shutil.rmtree(os.path.join(self.env.profile, "cache2"), ignore_errors=True) + + archiver = Archiver(self.scenario, self.env.profile, self.archive) + # the archive name is of the form + # profile[-vXYZ.x]-<platform>-<scenario>-<customization>.tgz + name = name % metadata + archive_name = os.path.join(self.archive, name) + dir = os.path.dirname(archive_name) + if not os.path.exists(dir): + os.makedirs(dir) + archiver.create_archive(archive_name) + logger.info("Archive created at %s" % archive_name) + statinfo = os.stat(archive_name) + logger.info("Current size is %d" % statinfo.st_size) + + logger.info("Extracting logs") + if "logs" in metadata: + logs = metadata.pop("logs") + for prefix, prefixed_logs in logs.items(): + for log in prefixed_logs: + content = obfuscate(log["content"])[1] + with open(os.path.join(dir, prefix + "-" + log["name"]), "wb") as f: + f.write(content.encode("utf-8")) + + if metadata.get("result", 0) != 0: + logger.info("The scenario returned a bad exit code") + raise Exception(metadata.get("result_message", "scenario error")) + self.changelog.append("update", **metadata) + + async def build_profile(self, device, headless): + scenario = self.scenario + profile = self.env.profile + customization_data = self.customization_data + + scenario_func = scenarii[scenario] + if scenario in customization_data.get("scenario", {}): + options = customization_data["scenario"][scenario] + logger.info("Loaded options for that scenario %s" % str(options)) + else: + options = {} + + # Adding general options + options["platform"] = self.env.target_platform + + if not self.force_new: + try: + custom_name = customization_data["name"] + get_profile(profile, self.env.target_platform, scenario, custom_name) + except ProfileNotFoundError: + # XXX we'll use a fresh profile for now + fresh_profile(profile, customization_data) + else: + fresh_profile(profile, customization_data) + + logger.info("Updating profile located at %r" % profile) + metadata = Metadata(profile) + + logger.info("Starting the Gecko app...") + adb_logs = self._log_filename("adb") + self.env.prepare(logfile=adb_logs) + geckodriver_logs = self._log_filename("geckodriver") + logger.info("Writing geckodriver logs in %s" % geckodriver_logs) + step = START + try: + firefox_instance = Firefox(**self.env.get_browser_args(headless)) + step = INIT_GECKODRIVER + with open(geckodriver_logs, "w") as glog: + geckodriver = self.env.get_geckodriver(log_file=glog) + step = START_SESSION + async with get_session(geckodriver, firefox_instance) as session: + step = START_SCENARIO + self.env.check_session(session) + logger.info("Running the %s scenario" % scenario) + metadata.update(await scenario_func(session, options)) + logger.info("%s scenario done." % scenario) + await close_extra_windows(session) + except Exception: + logger.error("%s scenario broke!" % scenario) + if step == START: + logger.info("Could not initialize the browser") + elif step == INIT_GECKODRIVER: + logger.info("Could not initialize Geckodriver") + elif step == START_SESSION: + logger.info( + "Could not start the session, check %s first" % geckodriver_logs + ) + else: + logger.info("Could not run the scenario, probably a faulty scenario") + raise + finally: + self.env.stop_browser() + for logfile in (adb_logs, geckodriver_logs): + if os.path.exists(logfile): + obfuscate_file(logfile) + self.env.collect_profile() + + # writing metadata + metadata.write( + name=self.scenario, + customization=self.customization_data["name"], + version=self.env.get_browser_version(), + platform=self.env.target_platform, + ) + + logger.info("Profile at %s.\nDone." % profile) + return metadata |