summaryrefslogtreecommitdiffstats
path: root/testing/condprofile/condprof/creator.py
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 17:32:43 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 17:32:43 +0000
commit6bf0a5cb5034a7e684dcc3500e841785237ce2dd (patch)
treea68f146d7fa01f0134297619fbe7e33db084e0aa /testing/condprofile/condprof/creator.py
parentInitial commit. (diff)
downloadthunderbird-upstream.tar.xz
thunderbird-upstream.zip
Adding upstream version 1:115.7.0.upstream/1%115.7.0upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '')
-rw-r--r--testing/condprofile/condprof/creator.py226
1 files changed, 226 insertions, 0 deletions
diff --git a/testing/condprofile/condprof/creator.py b/testing/condprofile/condprof/creator.py
new file mode 100644
index 0000000000..7e03f6a8da
--- /dev/null
+++ b/testing/condprofile/condprof/creator.py
@@ -0,0 +1,226 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+""" Creates or updates profiles.
+
+The profile creation works as following:
+
+For each scenario:
+
+- The latest indexed profile is picked on TC, if none we create a fresh profile
+- The scenario is done against it
+- The profile is uploaded on TC, replacing the previous one as the freshest
+
+For each platform we keep a changelog file that keep track of each update
+with the Task ID. That offers us the ability to get a profile from a specific
+date in the past.
+
+Artifacts are staying in TaskCluster for 3 months, and then they are removed,
+so the oldest profile we can get is 3 months old. Profiles are being updated
+continuously, so even after 3 months they are still getting "older".
+
+When Firefox changes its version, profiles from the previous version
+should work as expected. Each profile tarball comes with a metadata file
+that keep track of the Firefox version that was used and the profile age.
+"""
+import os
+import tempfile
+import shutil
+
+from arsenic import get_session
+from arsenic.browsers import Firefox
+
+from condprof.util import fresh_profile, logger, obfuscate_file, obfuscate, get_version
+from condprof.helpers import close_extra_windows
+from condprof.scenarii import scenarii
+from condprof.client import get_profile, ProfileNotFoundError
+from condprof.archiver import Archiver
+from condprof.customization import get_customization
+from condprof.metadata import Metadata
+
+
+START, INIT_GECKODRIVER, START_SESSION, START_SCENARIO = range(4)
+
+
+class ProfileCreator:
+ def __init__(
+ self,
+ scenario,
+ customization,
+ archive,
+ changelog,
+ force_new,
+ env,
+ skip_logs=False,
+ remote_test_root="/sdcard/test_root/",
+ ):
+ self.env = env
+ self.scenario = scenario
+ self.customization = customization
+ self.archive = archive
+ self.changelog = changelog
+ self.force_new = force_new
+ self.skip_logs = skip_logs
+ self.remote_test_root = remote_test_root
+ self.customization_data = get_customization(customization)
+ self.tmp_dir = None
+
+ # Make a temporary directory for the logs if an
+ # archive dir is not provided
+ if not self.archive:
+ self.tmp_dir = tempfile.mkdtemp()
+
+ def _log_filename(self, name):
+ filename = "%s-%s-%s.log" % (
+ name,
+ self.scenario,
+ self.customization_data["name"],
+ )
+ return os.path.join(self.archive or self.tmp_dir, filename)
+
+ async def run(self, headless=True):
+ logger.info(
+ "Building %s x %s" % (self.scenario, self.customization_data["name"])
+ )
+
+ if self.scenario in self.customization_data.get("ignore_scenario", []):
+ logger.info("Skipping (ignored scenario in that customization)")
+ return
+
+ filter_by_platform = self.customization_data.get("platforms")
+ if filter_by_platform and self.env.target_platform not in filter_by_platform:
+ logger.info("Skipping (ignored platform in that customization)")
+ return
+
+ with self.env.get_device(
+ 2828, verbose=True, remote_test_root=self.remote_test_root
+ ) as device:
+ try:
+ with self.env.get_browser():
+ metadata = await self.build_profile(device, headless)
+ except Exception:
+ raise
+ finally:
+ if not self.skip_logs:
+ self.env.dump_logs()
+
+ if not self.archive:
+ return
+
+ logger.info("Creating generic archive")
+ names = ["profile-%(platform)s-%(name)s-%(customization)s.tgz"]
+ if metadata["name"] == "full" and metadata["customization"] == "default":
+ names = [
+ "profile-%(platform)s-%(name)s-%(customization)s.tgz",
+ "profile-v%(version)s-%(platform)s-%(name)s-%(customization)s.tgz",
+ ]
+
+ for name in names:
+ # remove `cache` from profile
+ shutil.rmtree(os.path.join(self.env.profile, "cache"), ignore_errors=True)
+ shutil.rmtree(os.path.join(self.env.profile, "cache2"), ignore_errors=True)
+
+ archiver = Archiver(self.scenario, self.env.profile, self.archive)
+ # the archive name is of the form
+ # profile[-vXYZ.x]-<platform>-<scenario>-<customization>.tgz
+ name = name % metadata
+ archive_name = os.path.join(self.archive, name)
+ dir = os.path.dirname(archive_name)
+ if not os.path.exists(dir):
+ os.makedirs(dir)
+ archiver.create_archive(archive_name)
+ logger.info("Archive created at %s" % archive_name)
+ statinfo = os.stat(archive_name)
+ logger.info("Current size is %d" % statinfo.st_size)
+
+ logger.info("Extracting logs")
+ if "logs" in metadata:
+ logs = metadata.pop("logs")
+ for prefix, prefixed_logs in logs.items():
+ for log in prefixed_logs:
+ content = obfuscate(log["content"])[1]
+ with open(os.path.join(dir, prefix + "-" + log["name"]), "wb") as f:
+ f.write(content.encode("utf-8"))
+
+ if metadata.get("result", 0) != 0:
+ logger.info("The scenario returned a bad exit code")
+ raise Exception(metadata.get("result_message", "scenario error"))
+ self.changelog.append("update", **metadata)
+
+ async def build_profile(self, device, headless):
+ scenario = self.scenario
+ profile = self.env.profile
+ customization_data = self.customization_data
+
+ scenario_func = scenarii[scenario]
+ if scenario in customization_data.get("scenario", {}):
+ options = customization_data["scenario"][scenario]
+ logger.info("Loaded options for that scenario %s" % str(options))
+ else:
+ options = {}
+
+ # Adding general options
+ options["platform"] = self.env.target_platform
+
+ if not self.force_new:
+ try:
+ custom_name = customization_data["name"]
+ get_profile(profile, self.env.target_platform, scenario, custom_name)
+ except ProfileNotFoundError:
+ # XXX we'll use a fresh profile for now
+ fresh_profile(profile, customization_data)
+ else:
+ fresh_profile(profile, customization_data)
+
+ logger.info("Updating profile located at %r" % profile)
+ metadata = Metadata(profile)
+
+ logger.info("Starting the Gecko app...")
+ adb_logs = self._log_filename("adb")
+ self.env.prepare(logfile=adb_logs)
+ geckodriver_logs = self._log_filename("geckodriver")
+ logger.info("Writing geckodriver logs in %s" % geckodriver_logs)
+ step = START
+ try:
+ firefox_instance = Firefox(**self.env.get_browser_args(headless))
+ step = INIT_GECKODRIVER
+ with open(geckodriver_logs, "w") as glog:
+ geckodriver = self.env.get_geckodriver(log_file=glog)
+ step = START_SESSION
+ async with get_session(geckodriver, firefox_instance) as session:
+ step = START_SCENARIO
+ self.env.check_session(session)
+ logger.info("Running the %s scenario" % scenario)
+ metadata.update(await scenario_func(session, options))
+ logger.info("%s scenario done." % scenario)
+ await close_extra_windows(session)
+ except Exception:
+ logger.error("%s scenario broke!" % scenario)
+ if step == START:
+ logger.info("Could not initialize the browser")
+ elif step == INIT_GECKODRIVER:
+ logger.info("Could not initialize Geckodriver")
+ elif step == START_SESSION:
+ logger.info(
+ "Could not start the session, check %s first" % geckodriver_logs
+ )
+ else:
+ logger.info("Could not run the scenario, probably a faulty scenario")
+ raise
+ finally:
+ self.env.stop_browser()
+ for logfile in (adb_logs, geckodriver_logs):
+ if os.path.exists(logfile):
+ obfuscate_file(logfile)
+ self.env.collect_profile()
+
+ # writing metadata
+ metadata.write(
+ name=self.scenario,
+ customization=self.customization_data["name"],
+ version=self.env.get_browser_version(),
+ platform=self.env.target_platform,
+ )
+
+ logger.info("Profile at %s.\nDone." % profile)
+ return metadata