Adding upstream version 110.0.1.upstream/110.0.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 481 insertions, 0 deletions

diff --git a/testing/mozharness/scripts/desktop_l10n.py b/testing/mozharness/scripts/desktop_l10n.py
new file mode 100755
index 0000000000..6e401caa8b
--- /dev/null
+++ b/testing/mozharness/scripts/desktop_l10n.py
@@ -0,0 +1,481 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+"""desktop_l10n.py
+
+This script manages Desktop repacks for nightly builds.
+"""
+import glob
+import os
+import shlex
+import sys
+
+# load modules from parent dir
+sys.path.insert(1, os.path.dirname(sys.path[0]))  # noqa
+
+from mozharness.base.errors import MakefileErrorList
+from mozharness.base.script import BaseScript
+from mozharness.base.vcs.vcsbase import VCSMixin
+from mozharness.mozilla.automation import AutomationMixin
+from mozharness.mozilla.building.buildbase import (
+    MakeUploadOutputParser,
+    get_mozconfig_path,
+)
+from mozharness.mozilla.l10n.locales import LocalesMixin
+
+try:
+    import simplejson as json
+
+    assert json
+except ImportError:
+    import json
+
+
+# needed by _map
+SUCCESS = 0
+FAILURE = 1
+
+SUCCESS_STR = "Success"
+FAILURE_STR = "Failed"
+
+
+# DesktopSingleLocale {{{1
+class DesktopSingleLocale(LocalesMixin, AutomationMixin, VCSMixin, BaseScript):
+    """Manages desktop repacks"""
+
+    config_options = [
+        [
+            [
+                "--locale",
+            ],
+            {
+                "action": "extend",
+                "dest": "locales",
+                "type": "string",
+                "help": "Specify the locale(s) to sign and update. Optionally pass"
+                " revision separated by colon, en-GB:default.",
+            },
+        ],
+        [
+            [
+                "--tag-override",
+            ],
+            {
+                "action": "store",
+                "dest": "tag_override",
+                "type": "string",
+                "help": "Override the tags set for all repos",
+            },
+        ],
+        [
+            [
+                "--en-us-installer-url",
+            ],
+            {
+                "action": "store",
+                "dest": "en_us_installer_url",
+                "type": "string",
+                "help": "Specify the url of the en-us binary",
+            },
+        ],
+    ]
+
+    def __init__(self, require_config_file=True):
+        # fxbuild style:
+        buildscript_kwargs = {
+            "all_actions": [
+                "clone-locales",
+                "list-locales",
+                "setup",
+                "repack",
+                "summary",
+            ],
+            "config": {
+                "ignore_locales": ["en-US"],
+                "locales_dir": "browser/locales",
+                "log_name": "single_locale",
+                "hg_l10n_base": "https://hg.mozilla.org/l10n-central",
+            },
+        }
+
+        LocalesMixin.__init__(self)
+        BaseScript.__init__(
+            self,
+            config_options=self.config_options,
+            require_config_file=require_config_file,
+            **buildscript_kwargs
+        )
+
+        self.bootstrap_env = None
+        self.upload_env = None
+        self.upload_urls = {}
+        self.pushdate = None
+        # upload_files is a dictionary of files to upload, keyed by locale.
+        self.upload_files = {}
+
+    # Helper methods {{{2
+    def query_bootstrap_env(self):
+        """returns the env for repacks"""
+        if self.bootstrap_env:
+            return self.bootstrap_env
+        config = self.config
+        abs_dirs = self.query_abs_dirs()
+
+        bootstrap_env = self.query_env(
+            partial_env=config.get("bootstrap_env"), replace_dict=abs_dirs
+        )
+
+        bootstrap_env["L10NBASEDIR"] = abs_dirs["abs_l10n_dir"]
+        if self.query_is_nightly():
+            # we might set update_channel explicitly
+            if config.get("update_channel"):
+                update_channel = config["update_channel"]
+            else:  # Let's just give the generic channel based on branch.
+                update_channel = "nightly-%s" % (config["branch"],)
+            if not isinstance(update_channel, bytes):
+                update_channel = update_channel.encode("utf-8")
+            bootstrap_env["MOZ_UPDATE_CHANNEL"] = update_channel
+            self.info(
+                "Update channel set to: {}".format(bootstrap_env["MOZ_UPDATE_CHANNEL"])
+            )
+        self.bootstrap_env = bootstrap_env
+        return self.bootstrap_env
+
+    def _query_upload_env(self):
+        """returns the environment used for the upload step"""
+        if self.upload_env:
+            return self.upload_env
+        config = self.config
+
+        upload_env = self.query_env(partial_env=config.get("upload_env"))
+        # check if there are any extra option from the platform configuration
+        # and append them to the env
+
+        if "upload_env_extra" in config:
+            for extra in config["upload_env_extra"]:
+                upload_env[extra] = config["upload_env_extra"][extra]
+
+        self.upload_env = upload_env
+        return self.upload_env
+
+    def query_l10n_env(self):
+        l10n_env = self._query_upload_env().copy()
+        l10n_env.update(self.query_bootstrap_env())
+        return l10n_env
+
+    def _query_make_variable(self, variable, make_args=None):
+        """returns the value of make echo-variable-<variable>
+        it accepts extra make arguements (make_args)
+        """
+        dirs = self.query_abs_dirs()
+        make_args = make_args or []
+        target = ["echo-variable-%s" % variable] + make_args
+        cwd = dirs["abs_locales_dir"]
+        raw_output = self._get_output_from_make(
+            target, cwd=cwd, env=self.query_bootstrap_env()
+        )
+        # we want to log all the messages from make
+        output = []
+        for line in raw_output.split("\n"):
+            output.append(line.strip())
+        output = " ".join(output).strip()
+        self.info("echo-variable-%s: %s" % (variable, output))
+        return output
+
+    def _map(self, func, items):
+        """runs func for any item in items, calls the add_failure() for each
+        error. It assumes that function returns 0 when successful.
+        returns a two element tuple with (success_count, total_count)"""
+        success_count = 0
+        total_count = len(items)
+        name = func.__name__
+        for item in items:
+            result = func(item)
+            if result == SUCCESS:
+                #  success!
+                success_count += 1
+            else:
+                #  func failed...
+                message = "failure: %s(%s)" % (name, item)
+                self.add_failure(item, message)
+        return (success_count, total_count)
+
+    # Actions {{{2
+    def clone_locales(self):
+        self.pull_locale_source()
+
+    def setup(self):
+        """setup step"""
+        self._run_tooltool()
+        self._copy_mozconfig()
+        self._mach_configure()
+        self._run_make_in_config_dir()
+        self.make_wget_en_US()
+        self.make_unpack_en_US()
+
+    def _run_make_in_config_dir(self):
+        """this step creates nsinstall, needed my make_wget_en_US()"""
+        dirs = self.query_abs_dirs()
+        config_dir = os.path.join(dirs["abs_obj_dir"], "config")
+        env = self.query_bootstrap_env()
+        return self._make(target=["export"], cwd=config_dir, env=env)
+
+    def _copy_mozconfig(self):
+        """copies the mozconfig file into abs_src_dir/.mozconfig
+        and logs the content
+        """
+        config = self.config
+        dirs = self.query_abs_dirs()
+        src = get_mozconfig_path(self, config, dirs)
+        dst = os.path.join(dirs["abs_src_dir"], ".mozconfig")
+        self.copyfile(src, dst)
+        self.read_from_file(dst, verbose=True)
+
+    def _mach(self, target, env, halt_on_failure=True, output_parser=None):
+        dirs = self.query_abs_dirs()
+        mach = self._get_mach_executable()
+        return self.run_command(
+            mach + target,
+            halt_on_failure=True,
+            env=env,
+            cwd=dirs["abs_src_dir"],
+            output_parser=None,
+        )
+
+    def _mach_configure(self):
+        """calls mach configure"""
+        env = self.query_bootstrap_env()
+        target = ["configure"]
+        return self._mach(target=target, env=env)
+
+    def _get_mach_executable(self):
+        return [sys.executable, "mach"]
+
+    def _make(
+        self,
+        target,
+        cwd,
+        env,
+        error_list=MakefileErrorList,
+        halt_on_failure=True,
+        output_parser=None,
+    ):
+        """Runs make. Returns the exit code"""
+        make = ["make"]
+        if target:
+            make = make + target
+        return self.run_command(
+            make,
+            cwd=cwd,
+            env=env,
+            error_list=error_list,
+            halt_on_failure=halt_on_failure,
+            output_parser=output_parser,
+        )
+
+    def _get_output_from_make(
+        self, target, cwd, env, halt_on_failure=True, ignore_errors=False
+    ):
+        """runs make and returns the output of the command"""
+        return self.get_output_from_command(
+            ["make"] + target,
+            cwd=cwd,
+            env=env,
+            silent=True,
+            halt_on_failure=halt_on_failure,
+            ignore_errors=ignore_errors,
+        )
+
+    def make_unpack_en_US(self):
+        """wrapper for make unpack"""
+        config = self.config
+        dirs = self.query_abs_dirs()
+        env = self.query_bootstrap_env()
+        cwd = os.path.join(dirs["abs_obj_dir"], config["locales_dir"])
+        return self._make(target=["unpack"], cwd=cwd, env=env)
+
+    def make_wget_en_US(self):
+        """wrapper for make wget-en-US"""
+        env = self.query_bootstrap_env()
+        dirs = self.query_abs_dirs()
+        cwd = dirs["abs_locales_dir"]
+        return self._make(target=["wget-en-US"], cwd=cwd, env=env)
+
+    def make_upload(self, locale):
+        """wrapper for make upload command"""
+        env = self.query_l10n_env()
+        dirs = self.query_abs_dirs()
+        target = ["upload", "AB_CD=%s" % (locale)]
+        cwd = dirs["abs_locales_dir"]
+        parser = MakeUploadOutputParser(config=self.config, log_obj=self.log_obj)
+        retval = self._make(
+            target=target, cwd=cwd, env=env, halt_on_failure=False, output_parser=parser
+        )
+        if retval == SUCCESS:
+            self.info("Upload successful (%s)" % locale)
+            ret = SUCCESS
+        else:
+            self.error("failed to upload %s" % locale)
+            ret = FAILURE
+
+        if ret == FAILURE:
+            # If we failed above, we shouldn't even attempt a SIMPLE_NAME move
+            # even if we are configured to do so
+            return ret
+
+        # XXX Move the files to a SIMPLE_NAME format until we can enable
+        #     Simple names in the build system
+        if self.config.get("simple_name_move"):
+            # Assume an UPLOAD PATH
+            upload_target = self.config["upload_env"]["UPLOAD_PATH"]
+            target_path = os.path.join(upload_target, locale)
+            self.mkdir_p(target_path)
+            glob_name = "*.%s.*" % locale
+            matches = (
+                glob.glob(os.path.join(upload_target, glob_name))
+                + glob.glob(os.path.join(upload_target, "update", glob_name))
+                + glob.glob(os.path.join(upload_target, "*", "xpi", glob_name))
+                + glob.glob(os.path.join(upload_target, "install", "sea", glob_name))
+                + glob.glob(os.path.join(upload_target, "setup.exe"))
+                + glob.glob(os.path.join(upload_target, "setup-stub.exe"))
+            )
+            targets_exts = [
+                "tar.bz2",
+                "dmg",
+                "langpack.xpi",
+                "checksums",
+                "zip",
+                "installer.exe",
+                "installer-stub.exe",
+            ]
+            targets = [(".%s" % (ext,), "target.%s" % (ext,)) for ext in targets_exts]
+            targets.extend([(f, f) for f in ("setup.exe", "setup-stub.exe")])
+            for f in matches:
+                possible_targets = [
+                    (tail, target_file)
+                    for (tail, target_file) in targets
+                    if f.endswith(tail)
+                ]
+                if len(possible_targets) == 1:
+                    _, target_file = possible_targets[0]
+                    # Remove from list of available options for this locale
+                    targets.remove(possible_targets[0])
+                else:
+                    # wasn't valid (or already matched)
+                    raise RuntimeError(
+                        "Unexpected matching file name encountered: %s" % f
+                    )
+                self.move(os.path.join(f), os.path.join(target_path, target_file))
+            self.log("Converted uploads for %s to simple names" % locale)
+        return ret
+
+    def set_upload_files(self, locale):
+        # The tree doesn't have a good way of exporting the list of files
+        # created during locale generation, but we can grab them by echoing the
+        # UPLOAD_FILES variable for each locale.
+        env = self.query_l10n_env()
+        target = [
+            "echo-variable-UPLOAD_FILES",
+            "echo-variable-CHECKSUM_FILES",
+            "AB_CD=%s" % locale,
+        ]
+        dirs = self.query_abs_dirs()
+        cwd = dirs["abs_locales_dir"]
+        # Bug 1242771 - echo-variable-UPLOAD_FILES via mozharness fails when stderr is found
+        #    we should ignore stderr as unfortunately it's expected when parsing for values
+        output = self._get_output_from_make(
+            target=target, cwd=cwd, env=env, ignore_errors=True
+        )
+        self.info('UPLOAD_FILES is "%s"' % output)
+        files = shlex.split(output)
+        if not files:
+            self.error("failed to get upload file list for locale %s" % locale)
+            return FAILURE
+
+        self.upload_files[locale] = [
+            os.path.abspath(os.path.join(cwd, f)) for f in files
+        ]
+        return SUCCESS
+
+    def make_installers(self, locale):
+        """wrapper for make installers-(locale)"""
+        env = self.query_l10n_env()
+        env["PYTHONIOENCODING"] = "utf-8"
+        self._copy_mozconfig()
+        dirs = self.query_abs_dirs()
+        cwd = os.path.join(dirs["abs_locales_dir"])
+        target = [
+            "installers-%s" % locale,
+        ]
+        return self._make(target=target, cwd=cwd, env=env, halt_on_failure=False)
+
+    def repack_locale(self, locale):
+        """wraps the logic for make installers and generating
+        complete updates."""
+
+        # run make installers
+        if self.make_installers(locale) != SUCCESS:
+            self.error("make installers-%s failed" % (locale))
+            return FAILURE
+
+        # now try to upload the artifacts
+        if self.make_upload(locale):
+            self.error("make upload for locale %s failed!" % (locale))
+            return FAILURE
+
+        # set_upload_files() should be called after make upload, to make sure
+        # we have all files in place (checksums, etc)
+        if self.set_upload_files(locale):
+            self.error("failed to get list of files to upload for locale %s" % locale)
+            return FAILURE
+
+        return SUCCESS
+
+    def repack(self):
+        """creates the repacks and udpates"""
+        self._map(self.repack_locale, self.query_locales())
+
+    def _run_tooltool(self):
+        env = self.query_bootstrap_env()
+        config = self.config
+        dirs = self.query_abs_dirs()
+        manifest_src = os.environ.get("TOOLTOOL_MANIFEST")
+        if not manifest_src:
+            manifest_src = config.get("tooltool_manifest_src")
+        if not manifest_src:
+            return
+        python = sys.executable
+
+        cmd = [
+            python,
+            "-u",
+            os.path.join(dirs["abs_src_dir"], "mach"),
+            "artifact",
+            "toolchain",
+            "-v",
+            "--retry",
+            "4",
+            "--artifact-manifest",
+            os.path.join(dirs["abs_src_dir"], "toolchains.json"),
+        ]
+        if manifest_src:
+            cmd.extend(
+                [
+                    "--tooltool-manifest",
+                    os.path.join(dirs["abs_src_dir"], manifest_src),
+                ]
+            )
+        cache = config["bootstrap_env"].get("TOOLTOOL_CACHE")
+        if cache:
+            cmd.extend(["--cache-dir", cache])
+        self.info(str(cmd))
+        self.run_command(cmd, cwd=dirs["abs_src_dir"], halt_on_failure=True, env=env)
+
+
+# main {{{
+if __name__ == "__main__":
+    single_locale = DesktopSingleLocale()
+    single_locale.run_and_exit()