Adding upstream version 1:115.7.0.upstream/1%115.7.0upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 226 insertions, 0 deletions

diff --git a/testing/condprofile/condprof/creator.py b/testing/condprofile/condprof/creator.py
new file mode 100644
index 0000000000..7e03f6a8da
--- /dev/null
+++ b/testing/condprofile/condprof/creator.py
@@ -0,0 +1,226 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+""" Creates or updates profiles.
+
+The profile creation works as following:
+
+For each scenario:
+
+- The latest indexed profile is picked on TC, if none we create a fresh profile
+- The scenario is done against it
+- The profile is uploaded on TC, replacing the previous one as the freshest
+
+For each platform we keep a changelog file that keep track of each update
+with the Task ID. That offers us the ability to get a profile from a specific
+date in the past.
+
+Artifacts are staying in TaskCluster for 3 months, and then they are removed,
+so the oldest profile we can get is 3 months old. Profiles are being updated
+continuously, so even after 3 months they are still getting "older".
+
+When Firefox changes its version, profiles from the previous version
+should work as expected. Each profile tarball comes with a metadata file
+that keep track of the Firefox version that was used and the profile age.
+"""
+import os
+import tempfile
+import shutil
+
+from arsenic import get_session
+from arsenic.browsers import Firefox
+
+from condprof.util import fresh_profile, logger, obfuscate_file, obfuscate, get_version
+from condprof.helpers import close_extra_windows
+from condprof.scenarii import scenarii
+from condprof.client import get_profile, ProfileNotFoundError
+from condprof.archiver import Archiver
+from condprof.customization import get_customization
+from condprof.metadata import Metadata
+
+
+START, INIT_GECKODRIVER, START_SESSION, START_SCENARIO = range(4)
+
+
+class ProfileCreator:
+    def __init__(
+        self,
+        scenario,
+        customization,
+        archive,
+        changelog,
+        force_new,
+        env,
+        skip_logs=False,
+        remote_test_root="/sdcard/test_root/",
+    ):
+        self.env = env
+        self.scenario = scenario
+        self.customization = customization
+        self.archive = archive
+        self.changelog = changelog
+        self.force_new = force_new
+        self.skip_logs = skip_logs
+        self.remote_test_root = remote_test_root
+        self.customization_data = get_customization(customization)
+        self.tmp_dir = None
+
+        # Make a temporary directory for the logs if an
+        # archive dir is not provided
+        if not self.archive:
+            self.tmp_dir = tempfile.mkdtemp()
+
+    def _log_filename(self, name):
+        filename = "%s-%s-%s.log" % (
+            name,
+            self.scenario,
+            self.customization_data["name"],
+        )
+        return os.path.join(self.archive or self.tmp_dir, filename)
+
+    async def run(self, headless=True):
+        logger.info(
+            "Building %s x %s" % (self.scenario, self.customization_data["name"])
+        )
+
+        if self.scenario in self.customization_data.get("ignore_scenario", []):
+            logger.info("Skipping (ignored scenario in that customization)")
+            return
+
+        filter_by_platform = self.customization_data.get("platforms")
+        if filter_by_platform and self.env.target_platform not in filter_by_platform:
+            logger.info("Skipping (ignored platform in that customization)")
+            return
+
+        with self.env.get_device(
+            2828, verbose=True, remote_test_root=self.remote_test_root
+        ) as device:
+            try:
+                with self.env.get_browser():
+                    metadata = await self.build_profile(device, headless)
+            except Exception:
+                raise
+            finally:
+                if not self.skip_logs:
+                    self.env.dump_logs()
+
+        if not self.archive:
+            return
+
+        logger.info("Creating generic archive")
+        names = ["profile-%(platform)s-%(name)s-%(customization)s.tgz"]
+        if metadata["name"] == "full" and metadata["customization"] == "default":
+            names = [
+                "profile-%(platform)s-%(name)s-%(customization)s.tgz",
+                "profile-v%(version)s-%(platform)s-%(name)s-%(customization)s.tgz",
+            ]
+
+        for name in names:
+            # remove `cache` from profile
+            shutil.rmtree(os.path.join(self.env.profile, "cache"), ignore_errors=True)
+            shutil.rmtree(os.path.join(self.env.profile, "cache2"), ignore_errors=True)
+
+            archiver = Archiver(self.scenario, self.env.profile, self.archive)
+            # the archive name is of the form
+            # profile[-vXYZ.x]-<platform>-<scenario>-<customization>.tgz
+            name = name % metadata
+            archive_name = os.path.join(self.archive, name)
+            dir = os.path.dirname(archive_name)
+            if not os.path.exists(dir):
+                os.makedirs(dir)
+            archiver.create_archive(archive_name)
+            logger.info("Archive created at %s" % archive_name)
+            statinfo = os.stat(archive_name)
+            logger.info("Current size is %d" % statinfo.st_size)
+
+        logger.info("Extracting logs")
+        if "logs" in metadata:
+            logs = metadata.pop("logs")
+            for prefix, prefixed_logs in logs.items():
+                for log in prefixed_logs:
+                    content = obfuscate(log["content"])[1]
+                    with open(os.path.join(dir, prefix + "-" + log["name"]), "wb") as f:
+                        f.write(content.encode("utf-8"))
+
+        if metadata.get("result", 0) != 0:
+            logger.info("The scenario returned a bad exit code")
+            raise Exception(metadata.get("result_message", "scenario error"))
+        self.changelog.append("update", **metadata)
+
+    async def build_profile(self, device, headless):
+        scenario = self.scenario
+        profile = self.env.profile
+        customization_data = self.customization_data
+
+        scenario_func = scenarii[scenario]
+        if scenario in customization_data.get("scenario", {}):
+            options = customization_data["scenario"][scenario]
+            logger.info("Loaded options for that scenario %s" % str(options))
+        else:
+            options = {}
+
+        # Adding general options
+        options["platform"] = self.env.target_platform
+
+        if not self.force_new:
+            try:
+                custom_name = customization_data["name"]
+                get_profile(profile, self.env.target_platform, scenario, custom_name)
+            except ProfileNotFoundError:
+                # XXX we'll use a fresh profile for now
+                fresh_profile(profile, customization_data)
+        else:
+            fresh_profile(profile, customization_data)
+
+        logger.info("Updating profile located at %r" % profile)
+        metadata = Metadata(profile)
+
+        logger.info("Starting the Gecko app...")
+        adb_logs = self._log_filename("adb")
+        self.env.prepare(logfile=adb_logs)
+        geckodriver_logs = self._log_filename("geckodriver")
+        logger.info("Writing geckodriver logs in %s" % geckodriver_logs)
+        step = START
+        try:
+            firefox_instance = Firefox(**self.env.get_browser_args(headless))
+            step = INIT_GECKODRIVER
+            with open(geckodriver_logs, "w") as glog:
+                geckodriver = self.env.get_geckodriver(log_file=glog)
+                step = START_SESSION
+                async with get_session(geckodriver, firefox_instance) as session:
+                    step = START_SCENARIO
+                    self.env.check_session(session)
+                    logger.info("Running the %s scenario" % scenario)
+                    metadata.update(await scenario_func(session, options))
+                    logger.info("%s scenario done." % scenario)
+                    await close_extra_windows(session)
+        except Exception:
+            logger.error("%s scenario broke!" % scenario)
+            if step == START:
+                logger.info("Could not initialize the browser")
+            elif step == INIT_GECKODRIVER:
+                logger.info("Could not initialize Geckodriver")
+            elif step == START_SESSION:
+                logger.info(
+                    "Could not start the session, check %s first" % geckodriver_logs
+                )
+            else:
+                logger.info("Could not run the scenario, probably a faulty scenario")
+            raise
+        finally:
+            self.env.stop_browser()
+            for logfile in (adb_logs, geckodriver_logs):
+                if os.path.exists(logfile):
+                    obfuscate_file(logfile)
+        self.env.collect_profile()
+
+        # writing metadata
+        metadata.write(
+            name=self.scenario,
+            customization=self.customization_data["name"],
+            version=self.env.get_browser_version(),
+            platform=self.env.target_platform,
+        )
+
+        logger.info("Profile at %s.\nDone." % profile)
+        return metadata