# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. """ Creates or updates profiles. The profile creation works as following: For each scenario: - The latest indexed profile is picked on TC, if none we create a fresh profile - The scenario is done against it - The profile is uploaded on TC, replacing the previous one as the freshest For each platform we keep a changelog file that keep track of each update with the Task ID. That offers us the ability to get a profile from a specific date in the past. Artifacts are staying in TaskCluster for 3 months, and then they are removed, so the oldest profile we can get is 3 months old. Profiles are being updated continuously, so even after 3 months they are still getting "older". When Firefox changes its version, profiles from the previous version should work as expected. Each profile tarball comes with a metadata file that keep track of the Firefox version that was used and the profile age. """ import os import tempfile import shutil from arsenic import get_session from arsenic.browsers import Firefox from condprof.util import fresh_profile, logger, obfuscate_file, obfuscate, get_version from condprof.helpers import close_extra_windows from condprof.scenarii import scenarii from condprof.client import get_profile, ProfileNotFoundError from condprof.archiver import Archiver from condprof.customization import get_customization from condprof.metadata import Metadata START, INIT_GECKODRIVER, START_SESSION, START_SCENARIO = range(4) class ProfileCreator: def __init__( self, scenario, customization, archive, changelog, force_new, env, skip_logs=False, remote_test_root="/sdcard/test_root/", ): self.env = env self.scenario = scenario self.customization = customization self.archive = archive self.changelog = changelog self.force_new = force_new self.skip_logs = skip_logs self.remote_test_root = remote_test_root self.customization_data = get_customization(customization) self.tmp_dir = None # Make a temporary directory for the logs if an # archive dir is not provided if not self.archive: self.tmp_dir = tempfile.mkdtemp() def _log_filename(self, name): filename = "%s-%s-%s.log" % ( name, self.scenario, self.customization_data["name"], ) return os.path.join(self.archive or self.tmp_dir, filename) async def run(self, headless=True): logger.info( "Building %s x %s" % (self.scenario, self.customization_data["name"]) ) if self.scenario in self.customization_data.get("ignore_scenario", []): logger.info("Skipping (ignored scenario in that customization)") return filter_by_platform = self.customization_data.get("platforms") if filter_by_platform and self.env.target_platform not in filter_by_platform: logger.info("Skipping (ignored platform in that customization)") return with self.env.get_device( 2828, verbose=True, remote_test_root=self.remote_test_root ) as device: try: with self.env.get_browser(): metadata = await self.build_profile(device, headless) except Exception: raise finally: if not self.skip_logs: self.env.dump_logs() if not self.archive: return logger.info("Creating generic archive") names = ["profile-%(platform)s-%(name)s-%(customization)s.tgz"] if metadata["name"] == "full" and metadata["customization"] == "default": names = [ "profile-%(platform)s-%(name)s-%(customization)s.tgz", "profile-v%(version)s-%(platform)s-%(name)s-%(customization)s.tgz", ] for name in names: # remove `cache` from profile shutil.rmtree(os.path.join(self.env.profile, "cache"), ignore_errors=True) shutil.rmtree(os.path.join(self.env.profile, "cache2"), ignore_errors=True) archiver = Archiver(self.scenario, self.env.profile, self.archive) # the archive name is of the form # profile[-vXYZ.x]---.tgz name = name % metadata archive_name = os.path.join(self.archive, name) dir = os.path.dirname(archive_name) if not os.path.exists(dir): os.makedirs(dir) archiver.create_archive(archive_name) logger.info("Archive created at %s" % archive_name) statinfo = os.stat(archive_name) logger.info("Current size is %d" % statinfo.st_size) logger.info("Extracting logs") if "logs" in metadata: logs = metadata.pop("logs") for prefix, prefixed_logs in logs.items(): for log in prefixed_logs: content = obfuscate(log["content"])[1] with open(os.path.join(dir, prefix + "-" + log["name"]), "wb") as f: f.write(content.encode("utf-8")) if metadata.get("result", 0) != 0: logger.info("The scenario returned a bad exit code") raise Exception(metadata.get("result_message", "scenario error")) self.changelog.append("update", **metadata) async def build_profile(self, device, headless): scenario = self.scenario profile = self.env.profile customization_data = self.customization_data scenario_func = scenarii[scenario] if scenario in customization_data.get("scenario", {}): options = customization_data["scenario"][scenario] logger.info("Loaded options for that scenario %s" % str(options)) else: options = {} # Adding general options options["platform"] = self.env.target_platform if not self.force_new: try: custom_name = customization_data["name"] get_profile(profile, self.env.target_platform, scenario, custom_name) except ProfileNotFoundError: # XXX we'll use a fresh profile for now fresh_profile(profile, customization_data) else: fresh_profile(profile, customization_data) logger.info("Updating profile located at %r" % profile) metadata = Metadata(profile) logger.info("Starting the Gecko app...") adb_logs = self._log_filename("adb") self.env.prepare(logfile=adb_logs) geckodriver_logs = self._log_filename("geckodriver") logger.info("Writing geckodriver logs in %s" % geckodriver_logs) step = START try: firefox_instance = Firefox(**self.env.get_browser_args(headless)) step = INIT_GECKODRIVER with open(geckodriver_logs, "w") as glog: geckodriver = self.env.get_geckodriver(log_file=glog) step = START_SESSION async with get_session(geckodriver, firefox_instance) as session: step = START_SCENARIO self.env.check_session(session) logger.info("Running the %s scenario" % scenario) metadata.update(await scenario_func(session, options)) logger.info("%s scenario done." % scenario) await close_extra_windows(session) except Exception: logger.error("%s scenario broke!" % scenario) if step == START: logger.info("Could not initialize the browser") elif step == INIT_GECKODRIVER: logger.info("Could not initialize Geckodriver") elif step == START_SESSION: logger.info( "Could not start the session, check %s first" % geckodriver_logs ) else: logger.info("Could not run the scenario, probably a faulty scenario") raise finally: self.env.stop_browser() for logfile in (adb_logs, geckodriver_logs): if os.path.exists(logfile): obfuscate_file(logfile) self.env.collect_profile() # writing metadata metadata.write( name=self.scenario, customization=self.customization_data["name"], version=self.env.get_browser_version(), platform=self.env.target_platform, ) logger.info("Profile at %s.\nDone." % profile) return metadata