#!/usr/bin/env python # ***** BEGIN LICENSE BLOCK ***** # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. # ***** END LICENSE BLOCK ***** """desktop_l10n.py This script manages Desktop repacks for nightly builds. """ import glob import os import shlex import sys # load modules from parent dir sys.path.insert(1, os.path.dirname(sys.path[0])) # noqa from mozharness.base.errors import MakefileErrorList from mozharness.base.script import BaseScript from mozharness.base.vcs.vcsbase import VCSMixin from mozharness.mozilla.automation import AutomationMixin from mozharness.mozilla.building.buildbase import ( MakeUploadOutputParser, get_mozconfig_path, ) from mozharness.mozilla.l10n.locales import LocalesMixin try: import simplejson as json assert json except ImportError: import json # needed by _map SUCCESS = 0 FAILURE = 1 SUCCESS_STR = "Success" FAILURE_STR = "Failed" # DesktopSingleLocale {{{1 class DesktopSingleLocale(LocalesMixin, AutomationMixin, VCSMixin, BaseScript): """Manages desktop repacks""" config_options = [ [ [ "--locale", ], { "action": "extend", "dest": "locales", "type": "string", "help": "Specify the locale(s) to sign and update. Optionally pass" " revision separated by colon, en-GB:default.", }, ], [ [ "--tag-override", ], { "action": "store", "dest": "tag_override", "type": "string", "help": "Override the tags set for all repos", }, ], [ [ "--en-us-installer-url", ], { "action": "store", "dest": "en_us_installer_url", "type": "string", "help": "Specify the url of the en-us binary", }, ], ] def __init__(self, require_config_file=True): # fxbuild style: buildscript_kwargs = { "all_actions": [ "clone-locales", "list-locales", "setup", "repack", "summary", ], "config": { "ignore_locales": ["en-US"], "locales_dir": "browser/locales", "log_name": "single_locale", "hg_l10n_base": "https://hg.mozilla.org/l10n-central", }, } LocalesMixin.__init__(self) BaseScript.__init__( self, config_options=self.config_options, require_config_file=require_config_file, **buildscript_kwargs ) self.bootstrap_env = None self.upload_env = None self.upload_urls = {} self.pushdate = None # upload_files is a dictionary of files to upload, keyed by locale. self.upload_files = {} # Helper methods {{{2 def query_bootstrap_env(self): """returns the env for repacks""" if self.bootstrap_env: return self.bootstrap_env config = self.config abs_dirs = self.query_abs_dirs() bootstrap_env = self.query_env( partial_env=config.get("bootstrap_env"), replace_dict=abs_dirs ) bootstrap_env["L10NBASEDIR"] = abs_dirs["abs_l10n_dir"] if self.query_is_nightly(): # we might set update_channel explicitly if config.get("update_channel"): update_channel = config["update_channel"] else: # Let's just give the generic channel based on branch. update_channel = "nightly-%s" % (config["branch"],) if not isinstance(update_channel, bytes): update_channel = update_channel.encode("utf-8") bootstrap_env["MOZ_UPDATE_CHANNEL"] = update_channel self.info( "Update channel set to: {}".format(bootstrap_env["MOZ_UPDATE_CHANNEL"]) ) self.bootstrap_env = bootstrap_env return self.bootstrap_env def _query_upload_env(self): """returns the environment used for the upload step""" if self.upload_env: return self.upload_env config = self.config upload_env = self.query_env(partial_env=config.get("upload_env")) # check if there are any extra option from the platform configuration # and append them to the env if "upload_env_extra" in config: for extra in config["upload_env_extra"]: upload_env[extra] = config["upload_env_extra"][extra] self.upload_env = upload_env return self.upload_env def query_l10n_env(self): l10n_env = self._query_upload_env().copy() l10n_env.update(self.query_bootstrap_env()) return l10n_env def _query_make_variable(self, variable, make_args=None): """returns the value of make echo-variable- it accepts extra make arguements (make_args) """ dirs = self.query_abs_dirs() make_args = make_args or [] target = ["echo-variable-%s" % variable] + make_args cwd = dirs["abs_locales_dir"] raw_output = self._get_output_from_make( target, cwd=cwd, env=self.query_bootstrap_env() ) # we want to log all the messages from make output = [] for line in raw_output.split("\n"): output.append(line.strip()) output = " ".join(output).strip() self.info("echo-variable-%s: %s" % (variable, output)) return output def _map(self, func, items): """runs func for any item in items, calls the add_failure() for each error. It assumes that function returns 0 when successful. returns a two element tuple with (success_count, total_count)""" success_count = 0 total_count = len(items) name = func.__name__ for item in items: result = func(item) if result == SUCCESS: # success! success_count += 1 else: # func failed... message = "failure: %s(%s)" % (name, item) self.add_failure(item, message) return (success_count, total_count) # Actions {{{2 def clone_locales(self): self.pull_locale_source() def setup(self): """setup step""" self._run_tooltool() self._copy_mozconfig() self._mach_configure() self._run_make_in_config_dir() self.make_wget_en_US() self.make_unpack_en_US() def _run_make_in_config_dir(self): """this step creates nsinstall, needed my make_wget_en_US()""" dirs = self.query_abs_dirs() config_dir = os.path.join(dirs["abs_obj_dir"], "config") env = self.query_bootstrap_env() return self._make(target=["export"], cwd=config_dir, env=env) def _copy_mozconfig(self): """copies the mozconfig file into abs_src_dir/.mozconfig and logs the content """ config = self.config dirs = self.query_abs_dirs() src = get_mozconfig_path(self, config, dirs) dst = os.path.join(dirs["abs_src_dir"], ".mozconfig") self.copyfile(src, dst) self.read_from_file(dst, verbose=True) def _mach(self, target, env, halt_on_failure=True, output_parser=None): dirs = self.query_abs_dirs() mach = self._get_mach_executable() return self.run_command( mach + target, halt_on_failure=True, env=env, cwd=dirs["abs_src_dir"], output_parser=None, ) def _mach_configure(self): """calls mach configure""" env = self.query_bootstrap_env() target = ["configure"] return self._mach(target=target, env=env) def _get_mach_executable(self): return [sys.executable, "mach"] def _make( self, target, cwd, env, error_list=MakefileErrorList, halt_on_failure=True, output_parser=None, ): """Runs make. Returns the exit code""" make = ["make"] if target: make = make + target return self.run_command( make, cwd=cwd, env=env, error_list=error_list, halt_on_failure=halt_on_failure, output_parser=output_parser, ) def _get_output_from_make( self, target, cwd, env, halt_on_failure=True, ignore_errors=False ): """runs make and returns the output of the command""" return self.get_output_from_command( ["make"] + target, cwd=cwd, env=env, silent=True, halt_on_failure=halt_on_failure, ignore_errors=ignore_errors, ) def make_unpack_en_US(self): """wrapper for make unpack""" config = self.config dirs = self.query_abs_dirs() env = self.query_bootstrap_env() cwd = os.path.join(dirs["abs_obj_dir"], config["locales_dir"]) return self._make(target=["unpack"], cwd=cwd, env=env) def make_wget_en_US(self): """wrapper for make wget-en-US""" env = self.query_bootstrap_env() dirs = self.query_abs_dirs() cwd = dirs["abs_locales_dir"] return self._make(target=["wget-en-US"], cwd=cwd, env=env) def make_upload(self, locale): """wrapper for make upload command""" env = self.query_l10n_env() dirs = self.query_abs_dirs() target = ["upload", "AB_CD=%s" % (locale)] cwd = dirs["abs_locales_dir"] parser = MakeUploadOutputParser(config=self.config, log_obj=self.log_obj) retval = self._make( target=target, cwd=cwd, env=env, halt_on_failure=False, output_parser=parser ) if retval == SUCCESS: self.info("Upload successful (%s)" % locale) ret = SUCCESS else: self.error("failed to upload %s" % locale) ret = FAILURE if ret == FAILURE: # If we failed above, we shouldn't even attempt a SIMPLE_NAME move # even if we are configured to do so return ret # XXX Move the files to a SIMPLE_NAME format until we can enable # Simple names in the build system if self.config.get("simple_name_move"): # Assume an UPLOAD PATH upload_target = self.config["upload_env"]["UPLOAD_PATH"] target_path = os.path.join(upload_target, locale) self.mkdir_p(target_path) glob_name = "*.%s.*" % locale matches = ( glob.glob(os.path.join(upload_target, glob_name)) + glob.glob(os.path.join(upload_target, "update", glob_name)) + glob.glob(os.path.join(upload_target, "*", "xpi", glob_name)) + glob.glob(os.path.join(upload_target, "install", "sea", glob_name)) + glob.glob(os.path.join(upload_target, "setup.exe")) + glob.glob(os.path.join(upload_target, "setup-stub.exe")) ) targets_exts = [ "tar.bz2", "dmg", "langpack.xpi", "checksums", "zip", "installer.exe", "installer-stub.exe", ] targets = [(".%s" % (ext,), "target.%s" % (ext,)) for ext in targets_exts] targets.extend([(f, f) for f in ("setup.exe", "setup-stub.exe")]) for f in matches: possible_targets = [ (tail, target_file) for (tail, target_file) in targets if f.endswith(tail) ] if len(possible_targets) == 1: _, target_file = possible_targets[0] # Remove from list of available options for this locale targets.remove(possible_targets[0]) else: # wasn't valid (or already matched) raise RuntimeError( "Unexpected matching file name encountered: %s" % f ) self.move(os.path.join(f), os.path.join(target_path, target_file)) self.log("Converted uploads for %s to simple names" % locale) return ret def set_upload_files(self, locale): # The tree doesn't have a good way of exporting the list of files # created during locale generation, but we can grab them by echoing the # UPLOAD_FILES variable for each locale. env = self.query_l10n_env() target = [ "echo-variable-UPLOAD_FILES", "echo-variable-CHECKSUM_FILES", "AB_CD=%s" % locale, ] dirs = self.query_abs_dirs() cwd = dirs["abs_locales_dir"] # Bug 1242771 - echo-variable-UPLOAD_FILES via mozharness fails when stderr is found # we should ignore stderr as unfortunately it's expected when parsing for values output = self._get_output_from_make( target=target, cwd=cwd, env=env, ignore_errors=True ) self.info('UPLOAD_FILES is "%s"' % output) files = shlex.split(output) if not files: self.error("failed to get upload file list for locale %s" % locale) return FAILURE self.upload_files[locale] = [ os.path.abspath(os.path.join(cwd, f)) for f in files ] return SUCCESS def make_installers(self, locale): """wrapper for make installers-(locale)""" env = self.query_l10n_env() env["PYTHONIOENCODING"] = "utf-8" self._copy_mozconfig() dirs = self.query_abs_dirs() cwd = os.path.join(dirs["abs_locales_dir"]) target = [ "installers-%s" % locale, ] return self._make(target=target, cwd=cwd, env=env, halt_on_failure=False) def repack_locale(self, locale): """wraps the logic for make installers and generating complete updates.""" # run make installers if self.make_installers(locale) != SUCCESS: self.error("make installers-%s failed" % (locale)) return FAILURE # now try to upload the artifacts if self.make_upload(locale): self.error("make upload for locale %s failed!" % (locale)) return FAILURE # set_upload_files() should be called after make upload, to make sure # we have all files in place (checksums, etc) if self.set_upload_files(locale): self.error("failed to get list of files to upload for locale %s" % locale) return FAILURE return SUCCESS def repack(self): """creates the repacks and udpates""" self._map(self.repack_locale, self.query_locales()) def _run_tooltool(self): env = self.query_bootstrap_env() config = self.config dirs = self.query_abs_dirs() manifest_src = os.environ.get("TOOLTOOL_MANIFEST") if not manifest_src: manifest_src = config.get("tooltool_manifest_src") if not manifest_src: return python = sys.executable cmd = [ python, "-u", os.path.join(dirs["abs_src_dir"], "mach"), "artifact", "toolchain", "-v", "--retry", "4", "--artifact-manifest", os.path.join(dirs["abs_src_dir"], "toolchains.json"), ] if manifest_src: cmd.extend( [ "--tooltool-manifest", os.path.join(dirs["abs_src_dir"], manifest_src), ] ) cache = config["bootstrap_env"].get("TOOLTOOL_CACHE") if cache: cmd.extend(["--cache-dir", cache]) self.info(str(cmd)) self.run_command(cmd, cwd=dirs["abs_src_dir"], halt_on_failure=True, env=env) # main {{{ if __name__ == "__main__": single_locale = DesktopSingleLocale() single_locale.run_and_exit()