summaryrefslogtreecommitdiffstats
path: root/testing/mozharness/scripts/desktop_l10n.py
diff options
context:
space:
mode:
Diffstat (limited to '')
-rwxr-xr-xtesting/mozharness/scripts/desktop_l10n.py481
1 files changed, 481 insertions, 0 deletions
diff --git a/testing/mozharness/scripts/desktop_l10n.py b/testing/mozharness/scripts/desktop_l10n.py
new file mode 100755
index 0000000000..6e401caa8b
--- /dev/null
+++ b/testing/mozharness/scripts/desktop_l10n.py
@@ -0,0 +1,481 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+"""desktop_l10n.py
+
+This script manages Desktop repacks for nightly builds.
+"""
+import glob
+import os
+import shlex
+import sys
+
+# load modules from parent dir
+sys.path.insert(1, os.path.dirname(sys.path[0])) # noqa
+
+from mozharness.base.errors import MakefileErrorList
+from mozharness.base.script import BaseScript
+from mozharness.base.vcs.vcsbase import VCSMixin
+from mozharness.mozilla.automation import AutomationMixin
+from mozharness.mozilla.building.buildbase import (
+ MakeUploadOutputParser,
+ get_mozconfig_path,
+)
+from mozharness.mozilla.l10n.locales import LocalesMixin
+
+try:
+ import simplejson as json
+
+ assert json
+except ImportError:
+ import json
+
+
+# needed by _map
+SUCCESS = 0
+FAILURE = 1
+
+SUCCESS_STR = "Success"
+FAILURE_STR = "Failed"
+
+
+# DesktopSingleLocale {{{1
+class DesktopSingleLocale(LocalesMixin, AutomationMixin, VCSMixin, BaseScript):
+ """Manages desktop repacks"""
+
+ config_options = [
+ [
+ [
+ "--locale",
+ ],
+ {
+ "action": "extend",
+ "dest": "locales",
+ "type": "string",
+ "help": "Specify the locale(s) to sign and update. Optionally pass"
+ " revision separated by colon, en-GB:default.",
+ },
+ ],
+ [
+ [
+ "--tag-override",
+ ],
+ {
+ "action": "store",
+ "dest": "tag_override",
+ "type": "string",
+ "help": "Override the tags set for all repos",
+ },
+ ],
+ [
+ [
+ "--en-us-installer-url",
+ ],
+ {
+ "action": "store",
+ "dest": "en_us_installer_url",
+ "type": "string",
+ "help": "Specify the url of the en-us binary",
+ },
+ ],
+ ]
+
+ def __init__(self, require_config_file=True):
+ # fxbuild style:
+ buildscript_kwargs = {
+ "all_actions": [
+ "clone-locales",
+ "list-locales",
+ "setup",
+ "repack",
+ "summary",
+ ],
+ "config": {
+ "ignore_locales": ["en-US"],
+ "locales_dir": "browser/locales",
+ "log_name": "single_locale",
+ "hg_l10n_base": "https://hg.mozilla.org/l10n-central",
+ },
+ }
+
+ LocalesMixin.__init__(self)
+ BaseScript.__init__(
+ self,
+ config_options=self.config_options,
+ require_config_file=require_config_file,
+ **buildscript_kwargs
+ )
+
+ self.bootstrap_env = None
+ self.upload_env = None
+ self.upload_urls = {}
+ self.pushdate = None
+ # upload_files is a dictionary of files to upload, keyed by locale.
+ self.upload_files = {}
+
+ # Helper methods {{{2
+ def query_bootstrap_env(self):
+ """returns the env for repacks"""
+ if self.bootstrap_env:
+ return self.bootstrap_env
+ config = self.config
+ abs_dirs = self.query_abs_dirs()
+
+ bootstrap_env = self.query_env(
+ partial_env=config.get("bootstrap_env"), replace_dict=abs_dirs
+ )
+
+ bootstrap_env["L10NBASEDIR"] = abs_dirs["abs_l10n_dir"]
+ if self.query_is_nightly():
+ # we might set update_channel explicitly
+ if config.get("update_channel"):
+ update_channel = config["update_channel"]
+ else: # Let's just give the generic channel based on branch.
+ update_channel = "nightly-%s" % (config["branch"],)
+ if not isinstance(update_channel, bytes):
+ update_channel = update_channel.encode("utf-8")
+ bootstrap_env["MOZ_UPDATE_CHANNEL"] = update_channel
+ self.info(
+ "Update channel set to: {}".format(bootstrap_env["MOZ_UPDATE_CHANNEL"])
+ )
+ self.bootstrap_env = bootstrap_env
+ return self.bootstrap_env
+
+ def _query_upload_env(self):
+ """returns the environment used for the upload step"""
+ if self.upload_env:
+ return self.upload_env
+ config = self.config
+
+ upload_env = self.query_env(partial_env=config.get("upload_env"))
+ # check if there are any extra option from the platform configuration
+ # and append them to the env
+
+ if "upload_env_extra" in config:
+ for extra in config["upload_env_extra"]:
+ upload_env[extra] = config["upload_env_extra"][extra]
+
+ self.upload_env = upload_env
+ return self.upload_env
+
+ def query_l10n_env(self):
+ l10n_env = self._query_upload_env().copy()
+ l10n_env.update(self.query_bootstrap_env())
+ return l10n_env
+
+ def _query_make_variable(self, variable, make_args=None):
+ """returns the value of make echo-variable-<variable>
+ it accepts extra make arguements (make_args)
+ """
+ dirs = self.query_abs_dirs()
+ make_args = make_args or []
+ target = ["echo-variable-%s" % variable] + make_args
+ cwd = dirs["abs_locales_dir"]
+ raw_output = self._get_output_from_make(
+ target, cwd=cwd, env=self.query_bootstrap_env()
+ )
+ # we want to log all the messages from make
+ output = []
+ for line in raw_output.split("\n"):
+ output.append(line.strip())
+ output = " ".join(output).strip()
+ self.info("echo-variable-%s: %s" % (variable, output))
+ return output
+
+ def _map(self, func, items):
+ """runs func for any item in items, calls the add_failure() for each
+ error. It assumes that function returns 0 when successful.
+ returns a two element tuple with (success_count, total_count)"""
+ success_count = 0
+ total_count = len(items)
+ name = func.__name__
+ for item in items:
+ result = func(item)
+ if result == SUCCESS:
+ # success!
+ success_count += 1
+ else:
+ # func failed...
+ message = "failure: %s(%s)" % (name, item)
+ self.add_failure(item, message)
+ return (success_count, total_count)
+
+ # Actions {{{2
+ def clone_locales(self):
+ self.pull_locale_source()
+
+ def setup(self):
+ """setup step"""
+ self._run_tooltool()
+ self._copy_mozconfig()
+ self._mach_configure()
+ self._run_make_in_config_dir()
+ self.make_wget_en_US()
+ self.make_unpack_en_US()
+
+ def _run_make_in_config_dir(self):
+ """this step creates nsinstall, needed my make_wget_en_US()"""
+ dirs = self.query_abs_dirs()
+ config_dir = os.path.join(dirs["abs_obj_dir"], "config")
+ env = self.query_bootstrap_env()
+ return self._make(target=["export"], cwd=config_dir, env=env)
+
+ def _copy_mozconfig(self):
+ """copies the mozconfig file into abs_src_dir/.mozconfig
+ and logs the content
+ """
+ config = self.config
+ dirs = self.query_abs_dirs()
+ src = get_mozconfig_path(self, config, dirs)
+ dst = os.path.join(dirs["abs_src_dir"], ".mozconfig")
+ self.copyfile(src, dst)
+ self.read_from_file(dst, verbose=True)
+
+ def _mach(self, target, env, halt_on_failure=True, output_parser=None):
+ dirs = self.query_abs_dirs()
+ mach = self._get_mach_executable()
+ return self.run_command(
+ mach + target,
+ halt_on_failure=True,
+ env=env,
+ cwd=dirs["abs_src_dir"],
+ output_parser=None,
+ )
+
+ def _mach_configure(self):
+ """calls mach configure"""
+ env = self.query_bootstrap_env()
+ target = ["configure"]
+ return self._mach(target=target, env=env)
+
+ def _get_mach_executable(self):
+ return [sys.executable, "mach"]
+
+ def _make(
+ self,
+ target,
+ cwd,
+ env,
+ error_list=MakefileErrorList,
+ halt_on_failure=True,
+ output_parser=None,
+ ):
+ """Runs make. Returns the exit code"""
+ make = ["make"]
+ if target:
+ make = make + target
+ return self.run_command(
+ make,
+ cwd=cwd,
+ env=env,
+ error_list=error_list,
+ halt_on_failure=halt_on_failure,
+ output_parser=output_parser,
+ )
+
+ def _get_output_from_make(
+ self, target, cwd, env, halt_on_failure=True, ignore_errors=False
+ ):
+ """runs make and returns the output of the command"""
+ return self.get_output_from_command(
+ ["make"] + target,
+ cwd=cwd,
+ env=env,
+ silent=True,
+ halt_on_failure=halt_on_failure,
+ ignore_errors=ignore_errors,
+ )
+
+ def make_unpack_en_US(self):
+ """wrapper for make unpack"""
+ config = self.config
+ dirs = self.query_abs_dirs()
+ env = self.query_bootstrap_env()
+ cwd = os.path.join(dirs["abs_obj_dir"], config["locales_dir"])
+ return self._make(target=["unpack"], cwd=cwd, env=env)
+
+ def make_wget_en_US(self):
+ """wrapper for make wget-en-US"""
+ env = self.query_bootstrap_env()
+ dirs = self.query_abs_dirs()
+ cwd = dirs["abs_locales_dir"]
+ return self._make(target=["wget-en-US"], cwd=cwd, env=env)
+
+ def make_upload(self, locale):
+ """wrapper for make upload command"""
+ env = self.query_l10n_env()
+ dirs = self.query_abs_dirs()
+ target = ["upload", "AB_CD=%s" % (locale)]
+ cwd = dirs["abs_locales_dir"]
+ parser = MakeUploadOutputParser(config=self.config, log_obj=self.log_obj)
+ retval = self._make(
+ target=target, cwd=cwd, env=env, halt_on_failure=False, output_parser=parser
+ )
+ if retval == SUCCESS:
+ self.info("Upload successful (%s)" % locale)
+ ret = SUCCESS
+ else:
+ self.error("failed to upload %s" % locale)
+ ret = FAILURE
+
+ if ret == FAILURE:
+ # If we failed above, we shouldn't even attempt a SIMPLE_NAME move
+ # even if we are configured to do so
+ return ret
+
+ # XXX Move the files to a SIMPLE_NAME format until we can enable
+ # Simple names in the build system
+ if self.config.get("simple_name_move"):
+ # Assume an UPLOAD PATH
+ upload_target = self.config["upload_env"]["UPLOAD_PATH"]
+ target_path = os.path.join(upload_target, locale)
+ self.mkdir_p(target_path)
+ glob_name = "*.%s.*" % locale
+ matches = (
+ glob.glob(os.path.join(upload_target, glob_name))
+ + glob.glob(os.path.join(upload_target, "update", glob_name))
+ + glob.glob(os.path.join(upload_target, "*", "xpi", glob_name))
+ + glob.glob(os.path.join(upload_target, "install", "sea", glob_name))
+ + glob.glob(os.path.join(upload_target, "setup.exe"))
+ + glob.glob(os.path.join(upload_target, "setup-stub.exe"))
+ )
+ targets_exts = [
+ "tar.bz2",
+ "dmg",
+ "langpack.xpi",
+ "checksums",
+ "zip",
+ "installer.exe",
+ "installer-stub.exe",
+ ]
+ targets = [(".%s" % (ext,), "target.%s" % (ext,)) for ext in targets_exts]
+ targets.extend([(f, f) for f in ("setup.exe", "setup-stub.exe")])
+ for f in matches:
+ possible_targets = [
+ (tail, target_file)
+ for (tail, target_file) in targets
+ if f.endswith(tail)
+ ]
+ if len(possible_targets) == 1:
+ _, target_file = possible_targets[0]
+ # Remove from list of available options for this locale
+ targets.remove(possible_targets[0])
+ else:
+ # wasn't valid (or already matched)
+ raise RuntimeError(
+ "Unexpected matching file name encountered: %s" % f
+ )
+ self.move(os.path.join(f), os.path.join(target_path, target_file))
+ self.log("Converted uploads for %s to simple names" % locale)
+ return ret
+
+ def set_upload_files(self, locale):
+ # The tree doesn't have a good way of exporting the list of files
+ # created during locale generation, but we can grab them by echoing the
+ # UPLOAD_FILES variable for each locale.
+ env = self.query_l10n_env()
+ target = [
+ "echo-variable-UPLOAD_FILES",
+ "echo-variable-CHECKSUM_FILES",
+ "AB_CD=%s" % locale,
+ ]
+ dirs = self.query_abs_dirs()
+ cwd = dirs["abs_locales_dir"]
+ # Bug 1242771 - echo-variable-UPLOAD_FILES via mozharness fails when stderr is found
+ # we should ignore stderr as unfortunately it's expected when parsing for values
+ output = self._get_output_from_make(
+ target=target, cwd=cwd, env=env, ignore_errors=True
+ )
+ self.info('UPLOAD_FILES is "%s"' % output)
+ files = shlex.split(output)
+ if not files:
+ self.error("failed to get upload file list for locale %s" % locale)
+ return FAILURE
+
+ self.upload_files[locale] = [
+ os.path.abspath(os.path.join(cwd, f)) for f in files
+ ]
+ return SUCCESS
+
+ def make_installers(self, locale):
+ """wrapper for make installers-(locale)"""
+ env = self.query_l10n_env()
+ env["PYTHONIOENCODING"] = "utf-8"
+ self._copy_mozconfig()
+ dirs = self.query_abs_dirs()
+ cwd = os.path.join(dirs["abs_locales_dir"])
+ target = [
+ "installers-%s" % locale,
+ ]
+ return self._make(target=target, cwd=cwd, env=env, halt_on_failure=False)
+
+ def repack_locale(self, locale):
+ """wraps the logic for make installers and generating
+ complete updates."""
+
+ # run make installers
+ if self.make_installers(locale) != SUCCESS:
+ self.error("make installers-%s failed" % (locale))
+ return FAILURE
+
+ # now try to upload the artifacts
+ if self.make_upload(locale):
+ self.error("make upload for locale %s failed!" % (locale))
+ return FAILURE
+
+ # set_upload_files() should be called after make upload, to make sure
+ # we have all files in place (checksums, etc)
+ if self.set_upload_files(locale):
+ self.error("failed to get list of files to upload for locale %s" % locale)
+ return FAILURE
+
+ return SUCCESS
+
+ def repack(self):
+ """creates the repacks and udpates"""
+ self._map(self.repack_locale, self.query_locales())
+
+ def _run_tooltool(self):
+ env = self.query_bootstrap_env()
+ config = self.config
+ dirs = self.query_abs_dirs()
+ manifest_src = os.environ.get("TOOLTOOL_MANIFEST")
+ if not manifest_src:
+ manifest_src = config.get("tooltool_manifest_src")
+ if not manifest_src:
+ return
+ python = sys.executable
+
+ cmd = [
+ python,
+ "-u",
+ os.path.join(dirs["abs_src_dir"], "mach"),
+ "artifact",
+ "toolchain",
+ "-v",
+ "--retry",
+ "4",
+ "--artifact-manifest",
+ os.path.join(dirs["abs_src_dir"], "toolchains.json"),
+ ]
+ if manifest_src:
+ cmd.extend(
+ [
+ "--tooltool-manifest",
+ os.path.join(dirs["abs_src_dir"], manifest_src),
+ ]
+ )
+ cache = config["bootstrap_env"].get("TOOLTOOL_CACHE")
+ if cache:
+ cmd.extend(["--cache-dir", cache])
+ self.info(str(cmd))
+ self.run_command(cmd, cwd=dirs["abs_src_dir"], halt_on_failure=True, env=env)
+
+
+# main {{{
+if __name__ == "__main__":
+ single_locale = DesktopSingleLocale()
+ single_locale.run_and_exit()