226 lines
8.7 KiB
Python
226 lines
8.7 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
""" Creates or updates profiles.
|
|
|
|
The profile creation works as following:
|
|
|
|
For each scenario:
|
|
|
|
- The latest indexed profile is picked on TC, if none we create a fresh profile
|
|
- The scenario is done against it
|
|
- The profile is uploaded on TC, replacing the previous one as the freshest
|
|
|
|
For each platform we keep a changelog file that keep track of each update
|
|
with the Task ID. That offers us the ability to get a profile from a specific
|
|
date in the past.
|
|
|
|
Artifacts are staying in TaskCluster for 3 months, and then they are removed,
|
|
so the oldest profile we can get is 3 months old. Profiles are being updated
|
|
continuously, so even after 3 months they are still getting "older".
|
|
|
|
When Firefox changes its version, profiles from the previous version
|
|
should work as expected. Each profile tarball comes with a metadata file
|
|
that keep track of the Firefox version that was used and the profile age.
|
|
"""
|
|
import os
|
|
import tempfile
|
|
import shutil
|
|
|
|
from arsenic import get_session
|
|
from arsenic.browsers import Firefox
|
|
|
|
from condprof.util import fresh_profile, logger, obfuscate_file, obfuscate, get_version
|
|
from condprof.helpers import close_extra_windows
|
|
from condprof.scenarii import scenarii
|
|
from condprof.client import get_profile, ProfileNotFoundError
|
|
from condprof.archiver import Archiver
|
|
from condprof.customization import get_customization
|
|
from condprof.metadata import Metadata
|
|
|
|
|
|
START, INIT_GECKODRIVER, START_SESSION, START_SCENARIO = range(4)
|
|
|
|
|
|
class ProfileCreator:
|
|
def __init__(
|
|
self,
|
|
scenario,
|
|
customization,
|
|
archive,
|
|
changelog,
|
|
force_new,
|
|
env,
|
|
skip_logs=False,
|
|
remote_test_root="/sdcard/test_root/",
|
|
):
|
|
self.env = env
|
|
self.scenario = scenario
|
|
self.customization = customization
|
|
self.archive = archive
|
|
self.changelog = changelog
|
|
self.force_new = force_new
|
|
self.skip_logs = skip_logs
|
|
self.remote_test_root = remote_test_root
|
|
self.customization_data = get_customization(customization)
|
|
self.tmp_dir = None
|
|
|
|
# Make a temporary directory for the logs if an
|
|
# archive dir is not provided
|
|
if not self.archive:
|
|
self.tmp_dir = tempfile.mkdtemp()
|
|
|
|
def _log_filename(self, name):
|
|
filename = "%s-%s-%s.log" % (
|
|
name,
|
|
self.scenario,
|
|
self.customization_data["name"],
|
|
)
|
|
return os.path.join(self.archive or self.tmp_dir, filename)
|
|
|
|
async def run(self, headless=True):
|
|
logger.info(
|
|
"Building %s x %s" % (self.scenario, self.customization_data["name"])
|
|
)
|
|
|
|
if self.scenario in self.customization_data.get("ignore_scenario", []):
|
|
logger.info("Skipping (ignored scenario in that customization)")
|
|
return
|
|
|
|
filter_by_platform = self.customization_data.get("platforms")
|
|
if filter_by_platform and self.env.target_platform not in filter_by_platform:
|
|
logger.info("Skipping (ignored platform in that customization)")
|
|
return
|
|
|
|
with self.env.get_device(
|
|
2828, verbose=True, remote_test_root=self.remote_test_root
|
|
) as device:
|
|
try:
|
|
with self.env.get_browser():
|
|
metadata = await self.build_profile(device, headless)
|
|
except Exception:
|
|
raise
|
|
finally:
|
|
if not self.skip_logs:
|
|
self.env.dump_logs()
|
|
|
|
if not self.archive:
|
|
return
|
|
|
|
logger.info("Creating generic archive")
|
|
names = ["profile-%(platform)s-%(name)s-%(customization)s.tgz"]
|
|
if metadata["name"] == "full" and metadata["customization"] == "default":
|
|
names = [
|
|
"profile-%(platform)s-%(name)s-%(customization)s.tgz",
|
|
"profile-v%(version)s-%(platform)s-%(name)s-%(customization)s.tgz",
|
|
]
|
|
|
|
for name in names:
|
|
# remove `cache` from profile
|
|
shutil.rmtree(os.path.join(self.env.profile, "cache"), ignore_errors=True)
|
|
shutil.rmtree(os.path.join(self.env.profile, "cache2"), ignore_errors=True)
|
|
|
|
archiver = Archiver(self.scenario, self.env.profile, self.archive)
|
|
# the archive name is of the form
|
|
# profile[-vXYZ.x]-<platform>-<scenario>-<customization>.tgz
|
|
name = name % metadata
|
|
archive_name = os.path.join(self.archive, name)
|
|
dir = os.path.dirname(archive_name)
|
|
if not os.path.exists(dir):
|
|
os.makedirs(dir)
|
|
archiver.create_archive(archive_name)
|
|
logger.info("Archive created at %s" % archive_name)
|
|
statinfo = os.stat(archive_name)
|
|
logger.info("Current size is %d" % statinfo.st_size)
|
|
|
|
logger.info("Extracting logs")
|
|
if "logs" in metadata:
|
|
logs = metadata.pop("logs")
|
|
for prefix, prefixed_logs in logs.items():
|
|
for log in prefixed_logs:
|
|
content = obfuscate(log["content"])[1]
|
|
with open(os.path.join(dir, prefix + "-" + log["name"]), "wb") as f:
|
|
f.write(content.encode("utf-8"))
|
|
|
|
if metadata.get("result", 0) != 0:
|
|
logger.info("The scenario returned a bad exit code")
|
|
raise Exception(metadata.get("result_message", "scenario error"))
|
|
self.changelog.append("update", **metadata)
|
|
|
|
async def build_profile(self, device, headless):
|
|
scenario = self.scenario
|
|
profile = self.env.profile
|
|
customization_data = self.customization_data
|
|
|
|
scenario_func = scenarii[scenario]
|
|
if scenario in customization_data.get("scenario", {}):
|
|
options = customization_data["scenario"][scenario]
|
|
logger.info("Loaded options for that scenario %s" % str(options))
|
|
else:
|
|
options = {}
|
|
|
|
# Adding general options
|
|
options["platform"] = self.env.target_platform
|
|
|
|
if not self.force_new:
|
|
try:
|
|
custom_name = customization_data["name"]
|
|
get_profile(profile, self.env.target_platform, scenario, custom_name)
|
|
except ProfileNotFoundError:
|
|
# XXX we'll use a fresh profile for now
|
|
fresh_profile(profile, customization_data)
|
|
else:
|
|
fresh_profile(profile, customization_data)
|
|
|
|
logger.info("Updating profile located at %r" % profile)
|
|
metadata = Metadata(profile)
|
|
|
|
logger.info("Starting the Gecko app...")
|
|
adb_logs = self._log_filename("adb")
|
|
self.env.prepare(logfile=adb_logs)
|
|
geckodriver_logs = self._log_filename("geckodriver")
|
|
logger.info("Writing geckodriver logs in %s" % geckodriver_logs)
|
|
step = START
|
|
try:
|
|
firefox_instance = Firefox(**self.env.get_browser_args(headless))
|
|
step = INIT_GECKODRIVER
|
|
with open(geckodriver_logs, "w") as glog:
|
|
geckodriver = self.env.get_geckodriver(log_file=glog)
|
|
step = START_SESSION
|
|
async with get_session(geckodriver, firefox_instance) as session:
|
|
step = START_SCENARIO
|
|
self.env.check_session(session)
|
|
logger.info("Running the %s scenario" % scenario)
|
|
metadata.update(await scenario_func(session, options))
|
|
logger.info("%s scenario done." % scenario)
|
|
await close_extra_windows(session)
|
|
except Exception:
|
|
logger.error("%s scenario broke!" % scenario)
|
|
if step == START:
|
|
logger.info("Could not initialize the browser")
|
|
elif step == INIT_GECKODRIVER:
|
|
logger.info("Could not initialize Geckodriver")
|
|
elif step == START_SESSION:
|
|
logger.info(
|
|
"Could not start the session, check %s first" % geckodriver_logs
|
|
)
|
|
else:
|
|
logger.info("Could not run the scenario, probably a faulty scenario")
|
|
raise
|
|
finally:
|
|
self.env.stop_browser()
|
|
for logfile in (adb_logs, geckodriver_logs):
|
|
if os.path.exists(logfile):
|
|
obfuscate_file(logfile)
|
|
self.env.collect_profile()
|
|
|
|
# writing metadata
|
|
metadata.write(
|
|
name=self.scenario,
|
|
customization=self.customization_data["name"],
|
|
version=self.env.get_browser_version(),
|
|
platform=self.env.target_platform,
|
|
)
|
|
|
|
logger.info("Profile at %s.\nDone." % profile)
|
|
return metadata
|