diff options
Diffstat (limited to 'testing/mozharness')
28 files changed, 214 insertions, 730 deletions
diff --git a/testing/mozharness/configs/l10n_bumper/jamun.py b/testing/mozharness/configs/l10n_bumper/jamun.py deleted file mode 100644 index 355f82073a..0000000000 --- a/testing/mozharness/configs/l10n_bumper/jamun.py +++ /dev/null @@ -1,83 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import sys - -MULTI_REPO = "projects/jamun" -EXES = {} -if sys.platform.startswith("linux"): - EXES = { - # Get around the https warnings - "hg": [ - "/usr/local/bin/hg", - "--config", - "web.cacerts=/etc/pki/tls/certs/ca-bundle.crt", - ], - "hgtool.py": ["/usr/local/bin/hgtool.py"], - } - - -config = { - "log_name": "l10n_bumper", - "log_type": "multi", - "exes": EXES, - "gecko_pull_url": "https://hg.mozilla.org/{}".format(MULTI_REPO), - "gecko_push_url": "ssh://hg.mozilla.org/{}".format(MULTI_REPO), - "hg_user": "L10n Bumper Bot <release+l10nbumper@mozilla.com>", - "ssh_key": "~/.ssh/ffxbld_rsa", - "ssh_user": "ffxbld", - "vcs_share_base": "/builds/hg-shared", - "version_path": "browser/config/version.txt", - "status_path": ".l10n_bumper_status", - "bump_configs": [ - { - "path": "mobile/locales/l10n-changesets.json", - "format": "json", - "name": "Fennec l10n changesets", - "revision_url": "https://l10n.mozilla.org/shipping/l10n-changesets?av=fennec%(MAJOR_VERSION)s", - "platform_configs": [ - { - "platforms": ["android-arm", "android"], - "path": "mobile/android/locales/all-locales", - }, - { - "platforms": ["android-multilocale"], - "path": "mobile/android/locales/maemo-locales", - }, - ], - }, - { - "path": "browser/locales/l10n-changesets.json", - "format": "json", - "name": "Firefox l10n changesets", - "revision_url": "https://l10n.mozilla.org/shipping/l10n-changesets?av=fx%(MAJOR_VERSION)s", - "ignore_config": { - "ja": ["macosx64"], - "ja-JP-mac": ["linux", "linux64", "win32", "win64"], - }, - "platform_configs": [ - { - "platforms": ["linux64", "linux", "macosx64", "win32", "win64"], - "path": "browser/locales/shipped-locales", - "format": "shipped-locales", - } - ], - }, - { - "path": "browser/locales/central-changesets.json", - "format": "json", - "name": "Firefox l10n changesets", - "ignore_config": { - "ja": ["macosx64"], - "ja-JP-mac": ["linux", "linux64", "win32", "win64"], - }, - "platform_configs": [ - { - "platforms": ["linux64", "linux", "macosx64", "win32", "win64"], - "path": "browser/locales/all-locales", - } - ], - }, - ], -} diff --git a/testing/mozharness/configs/l10n_bumper/mozilla-beta.py b/testing/mozharness/configs/l10n_bumper/mozilla-beta.py deleted file mode 100644 index 4ba7add5de..0000000000 --- a/testing/mozharness/configs/l10n_bumper/mozilla-beta.py +++ /dev/null @@ -1,87 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import sys - -MULTI_REPO = "releases/mozilla-beta" -EXES = {} -if sys.platform.startswith("linux"): - EXES = { - # Get around the https warnings - "hg": [ - "/usr/local/bin/hg", - "--config", - "web.cacerts=/etc/pki/tls/certs/ca-bundle.crt", - ], - "hgtool.py": ["/usr/local/bin/hgtool.py"], - } - -config = { - "log_name": "l10n_bumper", - "log_type": "multi", - "exes": EXES, - "gecko_pull_url": "https://hg.mozilla.org/{}".format(MULTI_REPO), - "gecko_push_url": "ssh://hg.mozilla.org/{}".format(MULTI_REPO), - "hg_user": "L10n Bumper Bot <release+l10nbumper@mozilla.com>", - "ssh_key": "~/.ssh/ffxbld_rsa", - "ssh_user": "ffxbld", - "vcs_share_base": "/builds/hg-shared", - "version_path": "browser/config/version.txt", - "status_path": ".l10n_bumper_status", - "bump_configs": [ - { - "path": "mobile/locales/l10n-changesets.json", - "format": "json", - "name": "Fennec l10n changesets", - "revision_url": "https://l10n.mozilla.org/shipping/l10n-changesets?av=fennec%(MAJOR_VERSION)s", - "platform_configs": [ - { - "platforms": ["android-multilocale"], - "path": "mobile/android/locales/maemo-locales", - } - ], - }, - { - "path": "browser/locales/l10n-changesets.json", - "format": "json", - "name": "Firefox l10n changesets", - "revision_url": "https://l10n.mozilla.org/shipping/l10n-changesets?av=fx%(MAJOR_VERSION)s", - "ignore_config": { - "ja": ["macosx64", "macosx64-devedition"], - "ja-JP-mac": [ - "linux", - "linux-devedition", - "linux64", - "linux64-devedition", - "win32", - "win32-devedition", - "win64", - "win64-devedition", - "win64-aarch64", - "win64-aarch64-devedition", - ], - }, - "platform_configs": [ - { - "platforms": [ - "linux", - "linux-devedition", - "linux64", - "linux64-devedition", - "macosx64", - "macosx64-devedition", - "win32", - "win32-devedition", - "win64", - "win64-devedition", - "win64-aarch64", - "win64-aarch64-devedition", - ], - "path": "browser/locales/shipped-locales", - "format": "shipped-locales", - } - ], - }, - ], -} diff --git a/testing/mozharness/configs/l10n_bumper/mozilla-central.py b/testing/mozharness/configs/l10n_bumper/mozilla-central.py deleted file mode 100644 index b2bcc1b98f..0000000000 --- a/testing/mozharness/configs/l10n_bumper/mozilla-central.py +++ /dev/null @@ -1,88 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import sys - -MULTI_REPO = "mozilla-central" -EXES = {} -if sys.platform.startswith("linux"): - EXES = { - # Get around the https warnings - "hg": [ - "/usr/local/bin/hg", - "--config", - "web.cacerts=/etc/pki/tls/certs/ca-bundle.crt", - ], - "hgtool.py": ["/usr/local/bin/hgtool.py"], - } - -config = { - "log_name": "l10n_bumper", - "log_type": "multi", - "exes": EXES, - "gecko_pull_url": "https://hg.mozilla.org/{}".format(MULTI_REPO), - "gecko_push_url": "ssh://hg.mozilla.org/{}".format(MULTI_REPO), - "hg_user": "L10n Bumper Bot <release+l10nbumper@mozilla.com>", - "ssh_key": "~/.ssh/ffxbld_rsa", - "ssh_user": "ffxbld", - "vcs_share_base": "/builds/hg-shared", - "version_path": "browser/config/version.txt", - "status_path": ".l10n_bumper_status", - "bump_configs": [ - { - "path": "mobile/locales/l10n-changesets.json", - "format": "json", - "name": "Fennec l10n changesets", - "platform_configs": [ - { - "platforms": ["android-arm", "android"], - "path": "mobile/android/locales/all-locales", - }, - { - "platforms": ["android-multilocale"], - "path": "mobile/android/locales/maemo-locales", - }, - ], - }, - { - "path": "browser/locales/l10n-changesets.json", - "format": "json", - "name": "Firefox l10n changesets", - "ignore_config": { - "ja": ["macosx64", "macosx64-devedition"], - "ja-JP-mac": [ - "linux", - "linux-devedition", - "linux64", - "linux64-devedition", - "win32", - "win32-devedition", - "win64", - "win64-devedition", - "win64-aarch64", - "win64-aarch64-devedition", - ], - }, - "platform_configs": [ - { - "platforms": [ - "linux", - "linux-devedition", - "linux64", - "linux64-devedition", - "macosx64", - "macosx64-devedition", - "win32", - "win32-devedition", - "win64", - "win64-devedition", - "win64-aarch64", - "win64-aarch64-devedition", - ], - "path": "browser/locales/all-locales", - } - ], - }, - ], -} diff --git a/testing/mozharness/configs/l10n_bumper/mozilla-esr68.py b/testing/mozharness/configs/l10n_bumper/mozilla-esr68.py deleted file mode 100644 index 01a7c85e2d..0000000000 --- a/testing/mozharness/configs/l10n_bumper/mozilla-esr68.py +++ /dev/null @@ -1,46 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import sys - -MULTI_REPO = "releases/mozilla-esr68" -EXES = {} -if sys.platform.startswith("linux"): - EXES = { - # Get around the https warnings - "hg": [ - "/usr/local/bin/hg", - "--config", - "web.cacerts=/etc/pki/tls/certs/ca-bundle.crt", - ], - "hgtool.py": ["/usr/local/bin/hgtool.py"], - } - -config = { - "log_name": "l10n_bumper", - "log_type": "multi", - "exes": EXES, - "gecko_pull_url": "https://hg.mozilla.org/{}".format(MULTI_REPO), - "gecko_push_url": "ssh://hg.mozilla.org/{}".format(MULTI_REPO), - "hg_user": "L10n Bumper Bot <release+l10nbumper@mozilla.com>", - "ssh_key": "~/.ssh/ffxbld_rsa", - "ssh_user": "ffxbld", - "vcs_share_base": "/builds/hg-shared", - "version_path": "mobile/android/config/version-files/release/version.txt", - "status_path": ".l10n_bumper_status", - "bump_configs": [ - { - "path": "mobile/locales/l10n-changesets.json", - "format": "json", - "name": "Fennec l10n changesets", - "revision_url": "https://l10n.mozilla.org/shipping/l10n-changesets?av=fennec%(COMBINED_MAJOR_VERSION)s", - "platform_configs": [ - { - "platforms": ["android-multilocale"], - "path": "mobile/android/locales/maemo-locales", - } - ], - } - ], -} diff --git a/testing/mozharness/configs/marionette/windows_taskcluster_config.py b/testing/mozharness/configs/marionette/windows_taskcluster_config.py index a503885c66..324a89708e 100644 --- a/testing/mozharness/configs/marionette/windows_taskcluster_config.py +++ b/testing/mozharness/configs/marionette/windows_taskcluster_config.py @@ -17,8 +17,8 @@ DESKTOP_VISUALFX_THEME = { "Custom": 3, }.get("Best appearance") TASKBAR_AUTOHIDE_REG_PATH = { - "Windows 7": "HKCU:SOFTWARE\Microsoft\Windows\CurrentVersion\Explorer\StuckRects2", - "Windows 10": "HKCU:SOFTWARE\Microsoft\Windows\CurrentVersion\Explorer\StuckRects3", + "Windows 7": r"HKCU:SOFTWARE\Microsoft\Windows\CurrentVersion\Explorer\StuckRects2", + "Windows 10": r"HKCU:SOFTWARE\Microsoft\Windows\CurrentVersion\Explorer\StuckRects3", }.get("{} {}".format(platform.system(), platform.release())) ##### @@ -99,7 +99,7 @@ config = { "cmd": [ "powershell", "-command", - "\"&{{&Set-ItemProperty -Path 'HKCU:Software\Microsoft\Windows\CurrentVersion\Explorer\VisualEffects' -Name VisualFXSetting -Value {}}}\"".format( + "\"&{{&Set-ItemProperty -Path 'HKCU:Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\VisualEffects' -Name VisualFXSetting -Value {}}}\"".format( DESKTOP_VISUALFX_THEME ), ], @@ -112,7 +112,7 @@ config = { "cmd": [ "powershell", "-command", - "New-ItemProperty -Path 'HKCU:\Control Panel\Accessibility' -Name 'DynamicScrollbars' -Value 0", + r"New-ItemProperty -Path 'HKCU:\Control Panel\Accessibility' -Name 'DynamicScrollbars' -Value 0", ], "architectures": ["32bit", "64bit"], "halt_on_failure": False, diff --git a/testing/mozharness/configs/raptor/android_hw_external_browser_config.py b/testing/mozharness/configs/raptor/android_hw_external_browser_config.py new file mode 100644 index 0000000000..36f3ced4d8 --- /dev/null +++ b/testing/mozharness/configs/raptor/android_hw_external_browser_config.py @@ -0,0 +1,26 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os + +config = { + "log_name": "raptor", + "title": os.uname()[1].lower().split(".")[0], + "default_actions": [ + "clobber", + "download-and-extract", + "populate-webroot", + "create-virtualenv", + "install-chrome-android", + "install-chromium-distribution", + "run-tests", + ], + "tooltool_cache": "/builds/tooltool_cache", + "download_tooltool": True, + "hostutils_manifest_path": "testing/config/tooltool-manifests/linux64/hostutils.manifest", +} + +# raptor will pick these up in mitmproxy.py, doesn't use the mozharness config +os.environ["TOOLTOOLCACHE"] = config["tooltool_cache"] +os.environ["HOSTUTILS_MANIFEST_PATH"] = config["hostutils_manifest_path"] diff --git a/testing/mozharness/configs/raptor/linux_external_browser_config.py b/testing/mozharness/configs/raptor/linux_external_browser_config.py new file mode 100644 index 0000000000..9db0eb358c --- /dev/null +++ b/testing/mozharness/configs/raptor/linux_external_browser_config.py @@ -0,0 +1,23 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os + +VENV_PATH = "%s/build/venv" % os.getcwd() + +config = { + "log_name": "raptor", + "installer_path": "installer.exe", + "virtualenv_path": VENV_PATH, + "title": os.uname()[1].lower().split(".")[0], + "default_actions": [ + "clobber", + "download-and-extract", + "populate-webroot", + "create-virtualenv", + "install-chromium-distribution", + "run-tests", + ], + "tooltool_cache": "/builds/tooltool_cache", +} diff --git a/testing/mozharness/configs/raptor/mac_external_browser_config.py b/testing/mozharness/configs/raptor/mac_external_browser_config.py new file mode 100644 index 0000000000..2e5a83e10e --- /dev/null +++ b/testing/mozharness/configs/raptor/mac_external_browser_config.py @@ -0,0 +1,26 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os + +VENV_PATH = "%s/build/venv" % os.getcwd() + +config = { + "log_name": "raptor", + "installer_path": "installer.exe", + "virtualenv_path": VENV_PATH, + "title": os.uname()[1].lower().split(".")[0], + "default_actions": [ + "clobber", + "download-and-extract", + "populate-webroot", + "create-virtualenv", + "install-chromium-distribution", + "run-tests", + ], + "run_cmd_checks_enabled": True, + "preflight_run_cmd_suites": [], + "postflight_run_cmd_suites": [], + "tooltool_cache": "/builds/tooltool_cache", +} diff --git a/testing/mozharness/configs/raptor/windows_external_browser_config.py b/testing/mozharness/configs/raptor/windows_external_browser_config.py new file mode 100644 index 0000000000..56bb6708d6 --- /dev/null +++ b/testing/mozharness/configs/raptor/windows_external_browser_config.py @@ -0,0 +1,76 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import socket +import sys + +PYTHON = sys.executable +VENV_PATH = os.path.join(os.getcwd(), "build/venv") + +PYWIN32 = "pywin32==306" + +config = { + "log_name": "raptor", + "installer_path": "installer.exe", + "virtualenv_path": VENV_PATH, + "virtualenv_modules": [PYWIN32, "raptor", "mozinstall"], + "exes": { + "python": PYTHON, + "mozinstall": [ + "%s/scripts/python" % VENV_PATH, + "%s/scripts/mozinstall-script.py" % VENV_PATH, + ], + "hg": os.path.join(os.environ["PROGRAMFILES"], "Mercurial", "hg"), + "tooltool.py": [ + PYTHON, + os.path.join(os.environ["MOZILLABUILD"], "tooltool.py"), + ], + }, + "title": socket.gethostname().split(".")[0], + "default_actions": [ + "clobber", + "download-and-extract", + "populate-webroot", + "create-virtualenv", + "install-chromium-distribution", + "run-tests", + ], + "tooltool_cache": os.path.join("c:\\", "build", "tooltool_cache"), + "python3_manifest": { + "win32": "python3.manifest", + "win64": "python3_x64.manifest", + }, + "env": { + # python3 requires C runtime, found in firefox installation; see bug 1361732 + "PATH": "%(PATH)s;c:\\slave\\test\\build\\application\\firefox;" + }, + "run_cmd_checks_enabled": True, + "preflight_run_cmd_suites": [ + { + "name": "run mouse & screen adjustment script", + "cmd": [ + sys.executable, + os.path.join( + os.getcwd(), + "mozharness", + "external_tools", + "mouse_and_screen_resolution.py", + ), + "--configuration-file", + os.path.join( + os.getcwd(), + "mozharness", + "external_tools", + "machine-configuration.json", + ), + "--platform", + "win10-hw", + ], + "architectures": ["32bit", "64bit"], + "halt_on_failure": True, + "enabled": True, + } + ], +} diff --git a/testing/mozharness/configs/web_platform_tests/prod_config_windows_taskcluster.py b/testing/mozharness/configs/web_platform_tests/prod_config_windows_taskcluster.py index 7e84f69be4..537efb9632 100644 --- a/testing/mozharness/configs/web_platform_tests/prod_config_windows_taskcluster.py +++ b/testing/mozharness/configs/web_platform_tests/prod_config_windows_taskcluster.py @@ -20,8 +20,8 @@ DESKTOP_VISUALFX_THEME = { "Custom": 3, }.get("Best appearance") TASKBAR_AUTOHIDE_REG_PATH = { - "Windows 7": "HKCU:SOFTWARE\Microsoft\Windows\CurrentVersion\Explorer\StuckRects2", - "Windows 10": "HKCU:SOFTWARE\Microsoft\Windows\CurrentVersion\Explorer\StuckRects3", + "Windows 7": r"HKCU:SOFTWARE\Microsoft\Windows\CurrentVersion\Explorer\StuckRects2", + "Windows 10": r"HKCU:SOFTWARE\Microsoft\Windows\CurrentVersion\Explorer\StuckRects3", }.get("{} {}".format(platform.system(), platform.release())) ##### @@ -85,7 +85,7 @@ config = { "cmd": [ "powershell", "-command", - "\"&{{&Set-ItemProperty -Path 'HKCU:Software\Microsoft\Windows\CurrentVersion\Explorer\VisualEffects' -Name VisualFXSetting -Value {}}}\"".format( + "\"&{{&Set-ItemProperty -Path 'HKCU:Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\VisualEffects' -Name VisualFXSetting -Value {}}}\"".format( DESKTOP_VISUALFX_THEME ), ], @@ -98,7 +98,7 @@ config = { "cmd": [ "powershell", "-command", - "New-ItemProperty -Path 'HKCU:\Control Panel\Accessibility' -Name 'DynamicScrollbars' -Value 0", + r"New-ItemProperty -Path 'HKCU:\Control Panel\Accessibility' -Name 'DynamicScrollbars' -Value 0", ], "architectures": ["32bit", "64bit"], "halt_on_failure": False, diff --git a/testing/mozharness/examples/action_config_script.py b/testing/mozharness/examples/action_config_script.py index c86adc75d8..99022ce082 100755 --- a/testing/mozharness/examples/action_config_script.py +++ b/testing/mozharness/examples/action_config_script.py @@ -94,7 +94,7 @@ class ActionsConfigExample(BaseScript): def _ship1(self): self.info( - """ + r""" _~ _~ )_)_~ )_))_))_) @@ -106,7 +106,7 @@ class ActionsConfigExample(BaseScript): def _ship2(self): self.info( - """ + r""" _4 _4 _)_))_) _)_)_)_) diff --git a/testing/mozharness/external_tools/robustcheckout.py b/testing/mozharness/external_tools/robustcheckout.py index b5d2230211..d8af0c4316 100644 --- a/testing/mozharness/external_tools/robustcheckout.py +++ b/testing/mozharness/external_tools/robustcheckout.py @@ -407,7 +407,7 @@ def _docheckout( ui.warn(b"(shared store does not exist; deleting destination)\n") with timeit("removed_missing_shared_store", "remove-wdir"): destvfs.rmtree(forcibly=True) - elif not re.search(b"[a-f0-9]{40}/\.hg$", storepath.replace(b"\\", b"/")): + elif not re.search(rb"[a-f0-9]{40}/\.hg$", storepath.replace(b"\\", b"/")): ui.warn( b"(shared store does not belong to pooled storage; " b"deleting destination to improve efficiency)\n" diff --git a/testing/mozharness/mozharness/base/python.py b/testing/mozharness/mozharness/base/python.py index c98d01717e..05a4d3d16e 100644 --- a/testing/mozharness/mozharness/base/python.py +++ b/testing/mozharness/mozharness/base/python.py @@ -128,7 +128,6 @@ class VirtualenvMixin(object): method=None, requirements=None, optional=False, - two_pass=False, editable=False, ): """Register a module to be installed with the virtualenv. @@ -140,7 +139,7 @@ class VirtualenvMixin(object): applied. """ self._virtualenv_modules.append( - (name, url, method, requirements, optional, two_pass, editable) + (name, url, method, requirements, optional, editable) ) def query_virtualenv_path(self): @@ -291,6 +290,9 @@ class VirtualenvMixin(object): command = [pip, "install"] if no_deps: command += ["--no-deps"] + + command += ["--no-use-pep517"] + # To avoid timeouts with our pypi server, increase default timeout: # https://bugzilla.mozilla.org/show_bug.cgi?id=1007230#c802 command += ["--timeout", str(c.get("pip_timeout", 120))] @@ -625,19 +627,8 @@ class VirtualenvMixin(object): method, requirements, optional, - two_pass, editable, ) in self._virtualenv_modules: - if two_pass: - self.install_module( - module=module, - module_url=url, - install_method=method, - requirements=requirements or (), - optional=optional, - no_deps=True, - editable=editable, - ) self.install_module( module=module, module_url=url, @@ -1133,6 +1124,8 @@ class Python3Virtualenv(object): if c.get("find_links") and not c["pip_index"]: pip_args += ["--no-index"] + pip_args += ["--no-use-pep517"] + # Add --find-links pages to look at. Add --trusted-host automatically if # the host isn't secure. This allows modern versions of pip to connect # without requiring an override. diff --git a/testing/mozharness/mozharness/mozilla/building/buildbase.py b/testing/mozharness/mozharness/mozilla/building/buildbase.py index 777bd1e84e..9da3cf822b 100755 --- a/testing/mozharness/mozharness/mozilla/building/buildbase.py +++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py @@ -1486,8 +1486,10 @@ items from that key's value." if build_platform == "android-geckoview-docs": return main_platform = "android" + elif build_platform.startswith("ios"): + return else: - err = "Build platform {} didn't start with 'mac', 'linux', 'win', or 'android'".format( + err = "Build platform {} didn't start with 'mac', 'linux', 'win', 'android' or 'ios'".format( build_platform ) self.fatal(err) diff --git a/testing/mozharness/mozharness/mozilla/firefox/autoconfig.py b/testing/mozharness/mozharness/mozilla/firefox/autoconfig.py index 476277e661..8990bf9e04 100644 --- a/testing/mozharness/mozharness/mozilla/firefox/autoconfig.py +++ b/testing/mozharness/mozharness/mozilla/firefox/autoconfig.py @@ -56,7 +56,7 @@ def _autoconfig_path(fx_install_dir): def _cfg_file_path(fx_install_dir): - """ + r""" Windows: defaults\pref Mac: Firefox.app/Contents/Resources/defaults/pref Linux: defaults/pref diff --git a/testing/mozharness/mozharness/mozilla/mozbase.py b/testing/mozharness/mozharness/mozilla/mozbase.py index 552ffd850c..dcf62b8dec 100644 --- a/testing/mozharness/mozharness/mozilla/mozbase.py +++ b/testing/mozharness/mozharness/mozilla/mozbase.py @@ -29,4 +29,4 @@ class MozbaseMixin(object): "Could not find mozbase requirements file: {}".format(requirements) ) - self.register_virtualenv_module(requirements=[requirements], two_pass=True) + self.register_virtualenv_module(requirements=[requirements]) diff --git a/testing/mozharness/mozharness/mozilla/testing/raptor.py b/testing/mozharness/mozharness/mozilla/testing/raptor.py index d8571abe17..e4bca8301f 100644 --- a/testing/mozharness/mozharness/mozilla/testing/raptor.py +++ b/testing/mozharness/mozharness/mozilla/testing/raptor.py @@ -361,6 +361,17 @@ class Raptor( }, ], [ + ["--post-startup-delay"], + { + "dest": "post_startup_delay", + "type": "int", + "help": ( + "How long to wait (ms) after browser start-up before " + "starting the tests." + ), + }, + ], + [ ["--project"], { "action": "store", @@ -608,6 +619,15 @@ class Raptor( ), }, ], + [ + ["--screenshot-on-failure"], + { + "action": "store_true", + "dest": "screenshot_on_failure", + "default": False, + "help": "Take a screenshot when the test fails.", + }, + ], ] + testing_config_options + copy.deepcopy(code_coverage_config_options) @@ -732,6 +752,7 @@ class Raptor( self.browser_cycles = self.config.get("browser_cycles") self.clean = self.config.get("clean") self.page_timeout = self.config.get("page_timeout", None) + self.screenshot_on_failure = self.config.get("screenshot_on_failure") for (arg,), details in Raptor.browsertime_options: # Allow overriding defaults on the `./mach raptor-test ...` command-line. @@ -1054,6 +1075,15 @@ class Raptor( ) if self.config.get("page_timeout"): options.extend([f"--page-timeout={self.page_timeout}"]) + if self.config.get("post_startup_delay"): + options.extend( + [f"--post-startup-delay={self.config['post_startup_delay']}"] + ) + if ( + self.config.get("screenshot_on_failure", False) + or os.environ.get("MOZ_AUTOMATION", None) is not None + ): + options.extend(["--screenshot-on-failure"]) for (arg,), details in Raptor.browsertime_options: # Allow overriding defaults on the `./mach raptor-test ...` command-line @@ -1168,7 +1198,6 @@ class Raptor( ) self.register_virtualenv_module( requirements=[mozbase_requirements], - two_pass=True, editable=True, ) diff --git a/testing/mozharness/mozharness/mozilla/testing/talos.py b/testing/mozharness/mozharness/mozilla/testing/talos.py index 45d8fb59a8..3866c18a68 100755 --- a/testing/mozharness/mozharness/mozilla/testing/talos.py +++ b/testing/mozharness/mozharness/mozilla/testing/talos.py @@ -577,7 +577,7 @@ class Talos( if self.config.get("code_coverage", False): options.extend(["--code-coverage"]) if ( - self.config.get("--screenshot-on-failure", False) + self.config.get("screenshot_on_failure", False) or os.environ.get("MOZ_AUTOMATION", None) is not None ): options.extend(["--screenshot-on-failure"]) @@ -794,7 +794,6 @@ class Talos( ) self.register_virtualenv_module( requirements=[mozbase_requirements], - two_pass=True, editable=True, ) super(Talos, self).create_virtualenv() diff --git a/testing/mozharness/scripts/android_emulator_unittest.py b/testing/mozharness/scripts/android_emulator_unittest.py index 914fb7975f..7d798668e5 100644 --- a/testing/mozharness/scripts/android_emulator_unittest.py +++ b/testing/mozharness/scripts/android_emulator_unittest.py @@ -450,7 +450,7 @@ class AndroidEmulatorTest( "websocketprocessbridge_requirements_3.txt", ) if requirements: - self.register_virtualenv_module(requirements=[requirements], two_pass=True) + self.register_virtualenv_module(requirements=[requirements]) def download_and_extract(self): """ diff --git a/testing/mozharness/scripts/android_hardware_unittest.py b/testing/mozharness/scripts/android_hardware_unittest.py index 375f47fb32..fffccdf2ab 100644 --- a/testing/mozharness/scripts/android_hardware_unittest.py +++ b/testing/mozharness/scripts/android_hardware_unittest.py @@ -380,7 +380,7 @@ class AndroidHardwareTest( "websocketprocessbridge_requirements_3.txt", ) if requirements: - self.register_virtualenv_module(requirements=[requirements], two_pass=True) + self.register_virtualenv_module(requirements=[requirements]) def download_and_extract(self): """ diff --git a/testing/mozharness/scripts/awsy_script.py b/testing/mozharness/scripts/awsy_script.py index e54a342ca6..fed792a073 100644 --- a/testing/mozharness/scripts/awsy_script.py +++ b/testing/mozharness/scripts/awsy_script.py @@ -155,9 +155,7 @@ class AWSY(TestingMixin, MercurialScript, TooltoolMixin, CodeCoverageMixin): ] for requirements_file in requirements_files: - self.register_virtualenv_module( - requirements=[requirements_file], two_pass=True - ) + self.register_virtualenv_module(requirements=[requirements_file]) self.register_virtualenv_module("awsy", self.awsy_path) diff --git a/testing/mozharness/scripts/desktop_unittest.py b/testing/mozharness/scripts/desktop_unittest.py index 146f7bf54e..e42f507ff9 100755 --- a/testing/mozharness/scripts/desktop_unittest.py +++ b/testing/mozharness/scripts/desktop_unittest.py @@ -546,9 +546,7 @@ class DesktopUnittest(TestingMixin, MercurialScript, MozbaseMixin, CodeCoverageM ) for requirements_file in requirements_files: - self.register_virtualenv_module( - requirements=[requirements_file], two_pass=True - ) + self.register_virtualenv_module(requirements=[requirements_file]) _python_interp = self.query_exe("python") if "win" in self.platform_name() and os.path.exists(_python_interp): diff --git a/testing/mozharness/scripts/firefox_ui_tests.py b/testing/mozharness/scripts/firefox_ui_tests.py index 2177326483..09982d0a5b 100644 --- a/testing/mozharness/scripts/firefox_ui_tests.py +++ b/testing/mozharness/scripts/firefox_ui_tests.py @@ -128,7 +128,7 @@ class FirefoxUIFunctionalTests(TestingMixin, VCSToolsScript, CodeCoverageMixin): requirements = os.path.join( dirs["abs_test_install_dir"], "config", "firefox_ui_requirements.txt" ) - self.register_virtualenv_module(requirements=[requirements], two_pass=True) + self.register_virtualenv_module(requirements=[requirements]) def download_and_extract(self): """Override method from TestingMixin for more specific behavior.""" diff --git a/testing/mozharness/scripts/l10n_bumper.py b/testing/mozharness/scripts/l10n_bumper.py deleted file mode 100755 index e597d5386d..0000000000 --- a/testing/mozharness/scripts/l10n_bumper.py +++ /dev/null @@ -1,380 +0,0 @@ -#!/usr/bin/env python -# ***** BEGIN LICENSE BLOCK ***** -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at http://mozilla.org/MPL/2.0/. -# ***** END LICENSE BLOCK ***** -""" l10n_bumper.py - - Updates a gecko repo with up to date changesets from l10n.mozilla.org. - - Specifically, it updates l10n-changesets.json which is used by mobile releases. - - This is to allow for `mach taskgraph` to reference specific l10n revisions - without having to resort to task.extra or commandline base64 json hacks. -""" -import codecs -import os -import pprint -import sys -import time - -try: - import simplejson as json - - assert json -except ImportError: - import json - -sys.path.insert(1, os.path.dirname(sys.path[0])) - -from mozharness.base.errors import HgErrorList -from mozharness.base.log import FATAL -from mozharness.base.vcs.vcsbase import VCSScript - - -class L10nBumper(VCSScript): - config_options = [ - [ - [ - "--ignore-closed-tree", - ], - { - "action": "store_true", - "dest": "ignore_closed_tree", - "default": False, - "help": "Bump l10n changesets on a closed tree.", - }, - ], - [ - [ - "--build", - ], - { - "action": "store_false", - "dest": "dontbuild", - "default": True, - "help": "Trigger new builds on push.", - }, - ], - ] - - def __init__(self, require_config_file=True): - super(L10nBumper, self).__init__( - all_actions=[ - "clobber", - "check-treestatus", - "checkout-gecko", - "bump-changesets", - "push", - "push-loop", - ], - default_actions=[ - "push-loop", - ], - require_config_file=require_config_file, - config_options=self.config_options, - # Default config options - config={ - "treestatus_base_url": "https://treestatus.mozilla-releng.net", - "log_max_rotate": 99, - }, - ) - - # Helper methods {{{1 - def query_abs_dirs(self): - if self.abs_dirs: - return self.abs_dirs - - abs_dirs = super(L10nBumper, self).query_abs_dirs() - - abs_dirs.update( - { - "gecko_local_dir": os.path.join( - abs_dirs["abs_work_dir"], - self.config.get( - "gecko_local_dir", - os.path.basename(self.config["gecko_pull_url"]), - ), - ), - } - ) - self.abs_dirs = abs_dirs - return self.abs_dirs - - def hg_commit(self, path, repo_path, message): - """ - Commits changes in repo_path, with specified user and commit message - """ - user = self.config["hg_user"] - hg = self.query_exe("hg", return_type="list") - env = self.query_env(partial_env={"LANG": "en_US.UTF-8"}) - cmd = hg + ["add", path] - self.run_command(cmd, cwd=repo_path, env=env) - cmd = hg + ["commit", "-u", user, "-m", message] - self.run_command(cmd, cwd=repo_path, env=env) - - def hg_push(self, repo_path): - hg = self.query_exe("hg", return_type="list") - command = hg + [ - "push", - "-e", - "ssh -oIdentityFile=%s -l %s" - % ( - self.config["ssh_key"], - self.config["ssh_user"], - ), - "-r", - ".", - self.config["gecko_push_url"], - ] - status = self.run_command(command, cwd=repo_path, error_list=HgErrorList) - if status != 0: - # We failed; get back to a known state so we can either retry - # or fail out and continue later. - self.run_command( - hg - + ["--config", "extensions.mq=", "strip", "--no-backup", "outgoing()"], - cwd=repo_path, - ) - self.run_command(hg + ["up", "-C"], cwd=repo_path) - self.run_command( - hg + ["--config", "extensions.purge=", "purge", "--all"], cwd=repo_path - ) - return False - return True - - def _read_json(self, path): - contents = self.read_from_file(path) - try: - json_contents = json.loads(contents) - return json_contents - except ValueError: - self.error("%s is invalid json!" % path) - - def _read_version(self, path): - contents = self.read_from_file(path).split("\n")[0] - return contents.split(".") - - def _build_locale_map(self, old_contents, new_contents): - locale_map = {} - for key in old_contents: - if key not in new_contents: - locale_map[key] = "removed" - for k, v in new_contents.items(): - if old_contents.get(k, {}).get("revision") != v["revision"]: - locale_map[k] = v["revision"] - elif old_contents.get(k, {}).get("platforms") != v["platforms"]: - locale_map[k] = v["platforms"] - return locale_map - - def _build_platform_dict(self, bump_config): - dirs = self.query_abs_dirs() - repo_path = dirs["gecko_local_dir"] - platform_dict = {} - ignore_config = bump_config.get("ignore_config", {}) - for platform_config in bump_config["platform_configs"]: - path = os.path.join(repo_path, platform_config["path"]) - self.info( - "Reading %s for %s locales..." % (path, platform_config["platforms"]) - ) - contents = self.read_from_file(path) - for locale in contents.splitlines(): - # locale is 1st word in line in shipped-locales - if platform_config.get("format") == "shipped-locales": - locale = locale.split(" ")[0] - existing_platforms = set( - platform_dict.get(locale, {}).get("platforms", []) - ) - platforms = set(platform_config["platforms"]) - ignore_platforms = set(ignore_config.get(locale, [])) - platforms = (platforms | existing_platforms) - ignore_platforms - platform_dict[locale] = {"platforms": sorted(list(platforms))} - self.info("Built platform_dict:\n%s" % pprint.pformat(platform_dict)) - return platform_dict - - def _build_revision_dict(self, bump_config, version_list): - self.info("Building revision dict...") - platform_dict = self._build_platform_dict(bump_config) - revision_dict = {} - if bump_config.get("revision_url"): - repl_dict = { - "MAJOR_VERSION": version_list[0], - "COMBINED_MAJOR_VERSION": str( - int(version_list[0]) + int(version_list[1]) - ), - } - - url = bump_config["revision_url"] % repl_dict - path = self.download_file(url, error_level=FATAL) - revision_info = self.read_from_file(path) - self.info("Got %s" % revision_info) - for line in revision_info.splitlines(): - locale, revision = line.split(" ") - if locale in platform_dict: - revision_dict[locale] = platform_dict[locale] - revision_dict[locale]["revision"] = revision - else: - for k, v in platform_dict.items(): - v["revision"] = "default" - revision_dict[k] = v - self.info("revision_dict:\n%s" % pprint.pformat(revision_dict)) - return revision_dict - - def build_commit_message(self, name, locale_map): - comments = "" - approval_str = "r=release a=l10n-bump" - for locale, revision in sorted(locale_map.items()): - comments += "%s -> %s\n" % (locale, revision) - if self.config["dontbuild"]: - approval_str += " DONTBUILD" - if self.config["ignore_closed_tree"]: - approval_str += " CLOSED TREE" - message = "no bug - Bumping %s %s\n\n" % (name, approval_str) - message += comments - message = message.encode("utf-8") - return message - - def query_treestatus(self): - "Return True if we can land based on treestatus" - c = self.config - dirs = self.query_abs_dirs() - tree = c.get( - "treestatus_tree", os.path.basename(c["gecko_pull_url"].rstrip("/")) - ) - treestatus_url = "%s/trees/%s" % (c["treestatus_base_url"], tree) - treestatus_json = os.path.join(dirs["abs_work_dir"], "treestatus.json") - if not os.path.exists(dirs["abs_work_dir"]): - self.mkdir_p(dirs["abs_work_dir"]) - self.rmtree(treestatus_json) - - self.run_command( - ["curl", "--retry", "4", "-o", treestatus_json, treestatus_url], - throw_exception=True, - ) - - treestatus = self._read_json(treestatus_json) - if treestatus["result"]["status"] != "closed": - self.info( - "treestatus is %s - assuming we can land" - % repr(treestatus["result"]["status"]) - ) - return True - - return False - - # Actions {{{1 - def check_treestatus(self): - if not self.config["ignore_closed_tree"] and not self.query_treestatus(): - self.info("breaking early since treestatus is closed") - sys.exit(0) - - def checkout_gecko(self): - c = self.config - dirs = self.query_abs_dirs() - dest = dirs["gecko_local_dir"] - repos = [ - { - "repo": c["gecko_pull_url"], - "tag": c.get("gecko_tag", "default"), - "dest": dest, - "vcs": "hg", - } - ] - self.vcs_checkout_repos(repos) - - def bump_changesets(self): - dirs = self.query_abs_dirs() - repo_path = dirs["gecko_local_dir"] - version_path = os.path.join(repo_path, self.config["version_path"]) - changes = False - version_list = self._read_version(version_path) - for bump_config in self.config["bump_configs"]: - path = os.path.join(repo_path, bump_config["path"]) - # For now, assume format == 'json'. When we add desktop support, - # we may need to add flatfile support - if os.path.exists(path): - old_contents = self._read_json(path) - else: - old_contents = {} - - new_contents = self._build_revision_dict(bump_config, version_list) - - if new_contents == old_contents: - continue - # super basic sanity check - if not isinstance(new_contents, dict) or len(new_contents) < 5: - self.error( - "Cowardly refusing to land a broken-seeming changesets file!" - ) - continue - - # Write to disk - content_string = json.dumps( - new_contents, - sort_keys=True, - indent=4, - separators=(",", ": "), - ) - fh = codecs.open(path, encoding="utf-8", mode="w+") - fh.write(content_string + "\n") - fh.close() - - locale_map = self._build_locale_map(old_contents, new_contents) - - # Commit - message = self.build_commit_message(bump_config["name"], locale_map) - self.hg_commit(path, repo_path, message) - changes = True - return changes - - def push(self): - dirs = self.query_abs_dirs() - repo_path = dirs["gecko_local_dir"] - return self.hg_push(repo_path) - - def push_loop(self): - max_retries = 5 - for _ in range(max_retries): - changed = False - if not self.config["ignore_closed_tree"] and not self.query_treestatus(): - # Tree is closed; exit early to avoid a bunch of wasted time - self.info("breaking early since treestatus is closed") - break - - self.checkout_gecko() - if self.bump_changesets(): - changed = True - - if not changed: - # Nothing changed, we're all done - self.info("No changes - all done") - break - - if self.push(): - # We did it! Hurray! - self.info("Great success!") - break - # If we're here, then the push failed. It also stripped any - # outgoing commits, so we should be in a pristine state again - # Empty our local cache of manifests so they get loaded again next - # time through this loop. This makes sure we get fresh upstream - # manifests, and avoids problems like bug 979080 - self.device_manifests = {} - - # Sleep before trying again - self.info("Sleeping 60 before trying again") - time.sleep(60) - else: - self.fatal("Didn't complete successfully (hit max_retries)") - - # touch status file for nagios - dirs = self.query_abs_dirs() - status_path = os.path.join(dirs["base_work_dir"], self.config["status_path"]) - self._touch_file(status_path) - - -# __main__ {{{1 -if __name__ == "__main__": - bumper = L10nBumper() - bumper.run_and_exit() diff --git a/testing/mozharness/scripts/marionette.py b/testing/mozharness/scripts/marionette.py index ba59f8a11b..82236bcf91 100755 --- a/testing/mozharness/scripts/marionette.py +++ b/testing/mozharness/scripts/marionette.py @@ -257,7 +257,7 @@ class MarionetteTest(TestingMixin, MercurialScript, TransferMixin, CodeCoverageM "Could not find marionette requirements file: {}".format(requirements) ) - self.register_virtualenv_module(requirements=[requirements], two_pass=True) + self.register_virtualenv_module(requirements=[requirements]) def _get_test_suite(self, is_emulator): """ diff --git a/testing/mozharness/scripts/openh264_build.py b/testing/mozharness/scripts/openh264_build.py index a0965af0af..ed9506edc1 100755 --- a/testing/mozharness/scripts/openh264_build.py +++ b/testing/mozharness/scripts/openh264_build.py @@ -363,7 +363,7 @@ class OpenH264Build(TransferMixin, VCSScript, TooltoolMixin): to_package = [] for f in glob.glob(os.path.join(srcdir, "*gmpopenh264*")): if not re.search( - "(?:lib)?gmpopenh264(?!\.\d)\.(?:dylib|so|dll|info)(?!\.\d)", f + r"(?:lib)?gmpopenh264(?!\.\d)\.(?:dylib|so|dll|info)(?!\.\d)", f ): # Don't package unnecessary zip bloat # Blocks things like libgmpopenh264.2.dylib and libgmpopenh264.so.1 diff --git a/testing/mozharness/scripts/telemetry/telemetry_client.py b/testing/mozharness/scripts/telemetry/telemetry_client.py index 05a7b967ad..b1c8895733 100755 --- a/testing/mozharness/scripts/telemetry/telemetry_client.py +++ b/testing/mozharness/scripts/telemetry/telemetry_client.py @@ -138,7 +138,7 @@ class TelemetryTests(TestingMixin, VCSToolsScript, CodeCoverageMixin): "config", "telemetry_tests_requirements.txt", ) - self.register_virtualenv_module(requirements=[requirements], two_pass=True) + self.register_virtualenv_module(requirements=[requirements]) def query_abs_dirs(self): if self.abs_dirs: diff --git a/testing/mozharness/scripts/web_platform_tests.py b/testing/mozharness/scripts/web_platform_tests.py index 83235633b7..e4c3c3983c 100755 --- a/testing/mozharness/scripts/web_platform_tests.py +++ b/testing/mozharness/scripts/web_platform_tests.py @@ -305,7 +305,7 @@ class WebPlatformTest(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidM dirs["abs_test_install_dir"], "config", "marionette_requirements.txt" ) - self.register_virtualenv_module(requirements=[requirements], two_pass=True) + self.register_virtualenv_module(requirements=[requirements]) webtransport_requirements = os.path.join( dirs["abs_test_install_dir"], @@ -316,9 +316,7 @@ class WebPlatformTest(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidM "requirements.txt", ) - self.register_virtualenv_module( - requirements=[webtransport_requirements], two_pass=True - ) + self.register_virtualenv_module(requirements=[webtransport_requirements]) def _query_geckodriver(self): path = None |