Merging upstream version 127.0.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

18 files changed, 51 insertions, 44 deletions

diff --git a/testing/mozbase/manifestparser/manifestparser/ini.py b/testing/mozbase/manifestparser/manifestparser/ini.py
index b5ffe7a2f0..68f6023244 100644
--- a/testing/mozbase/manifestparser/manifestparser/ini.py
+++ b/testing/mozbase/manifestparser/manifestparser/ini.py
@@ -30,6 +30,7 @@ def read_ini(
     strict=True,
     handle_defaults=True,
     document=False,
+    add_line_no=False,
 ):
     """
     read an .ini file and return a list of [(section, values)]
@@ -40,6 +41,7 @@ def read_ini(
     - separators : strings that denote key, value separation in order
     - strict : whether to be strict about parsing
     - handle_defaults : whether to incorporate defaults into each section
+    - add_line_no: whether to include the line number that points to the test in the generated ini file.
     """
 
     # variables
diff --git a/testing/mozbase/manifestparser/manifestparser/manifestparser.py b/testing/mozbase/manifestparser/manifestparser/manifestparser.py
index 63eaeefe05..d8df2baf36 100644
--- a/testing/mozbase/manifestparser/manifestparser/manifestparser.py
+++ b/testing/mozbase/manifestparser/manifestparser/manifestparser.py
@@ -56,6 +56,7 @@ class ManifestParser(object):
         handle_defaults=True,
         use_toml=True,
         document=False,
+        add_line_no=False,
     ):
         """Creates a ManifestParser from the given manifest files.
 
@@ -82,6 +83,7 @@ class ManifestParser(object):
                                 variable in this case.
         :param use_toml: If True *.toml configration files will be used iff present in the same location as *.ini files (applies to included files as well). If False only *.ini files will be considered. (defaults to True)
         :param document: If True *.toml configration will preserve the parsed document from `tomlkit` in self.source_documents[filename] (defaults to False)
+        :param add_line_no: If True, the *.toml configuration will add the line number where the test name appears in the file to the parsed document. Also, the document should be set to True. (defaults to False)
         """
         self._defaults = defaults or {}
         self.tests = []
@@ -95,6 +97,7 @@ class ManifestParser(object):
         self._handle_defaults = handle_defaults
         self.use_toml = use_toml
         self.document = document
+        self.add_line_no = add_line_no
         self.logger = Logger()
         if manifests:
             self.read(*manifests)
@@ -229,6 +232,7 @@ class ManifestParser(object):
             strict=self.strict,
             handle_defaults=self._handle_defaults,
             document=self.document,
+            add_line_no=self.add_line_no,
         )
         if filename is not None:
             self.source_documents[filename] = document
diff --git a/testing/mozbase/manifestparser/manifestparser/toml.py b/testing/mozbase/manifestparser/manifestparser/toml.py
index 4defd99156..a3e9511994 100644
--- a/testing/mozbase/manifestparser/manifestparser/toml.py
+++ b/testing/mozbase/manifestparser/manifestparser/toml.py
@@ -82,6 +82,7 @@ def read_toml(
     strict=True,
     handle_defaults=True,
     document=False,
+    add_line_no=False,
 ):
     """
     read a .toml file and return a list of [(section, values)]
@@ -93,6 +94,7 @@ def read_toml(
     - strict : whether to be strict about parsing
     - handle_defaults : whether to incorporate defaults into each section
     - document: read TOML with tomlkit and return source in test["document"]
+    - add_line_no: add the line number where the test name appears in the file to the source. Also, the document variable must be set to True for this flag to work. (This is used only to generate the documentation)
     """
 
     # variables
@@ -163,12 +165,12 @@ def read_toml(
         # merge combined defaults into each section
         sections = [(i, combine_fields(defaults, j)) for i, j in sections]
 
-    if document:
+    if document and add_line_no:
         # Take the line where the test name appears in the file.
         for i, _ in enumerate(sections):
             line = contents.split(sections[i][0])[0].count(os.linesep) + 1
             manifest.setdefault(sections[i][0], {})["lineno"] = str(line)
-    else:
+    elif not document:
         manifest = None
 
     return sections, defaults, manifest
diff --git a/testing/mozbase/manifestparser/tests/test_manifestparser.py b/testing/mozbase/manifestparser/tests/test_manifestparser.py
index ddbc6ecc79..f1774cfffb 100755
--- a/testing/mozbase/manifestparser/tests/test_manifestparser.py
+++ b/testing/mozbase/manifestparser/tests/test_manifestparser.py
@@ -5,7 +5,6 @@
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os
-import re
 import shutil
 import tempfile
 import unittest
@@ -621,12 +620,6 @@ yellow = submarine
         after = "edit-manifest-after.toml"
         after_path = os.path.join(here, after)
         after_str = open(after_path, "r", encoding="utf-8").read()
-
-        # Define the regex pattern to match lines containing 'lineno'
-        pattern = re.compile(r"^.*lineno.*$\n?", re.MULTILINE)
-        # Remove lines containing 'lineno'
-        manifest_str = re.sub(pattern, "", manifest_str)
-
         assert manifest_str == after_str
 
 
diff --git a/testing/mozbase/mozdebug/setup.py b/testing/mozbase/mozdebug/setup.py
index 2e28924fad..a2f406a257 100644
--- a/testing/mozbase/mozdebug/setup.py
+++ b/testing/mozbase/mozdebug/setup.py
@@ -4,7 +4,7 @@
 
 from setuptools import setup
 
-PACKAGE_VERSION = "0.3.0"
+PACKAGE_VERSION = "0.3.1"
 DEPS = ["mozinfo"]
 
 
diff --git a/testing/mozbase/mozdevice/setup.py b/testing/mozbase/mozdevice/setup.py
index 91ce63d9f6..5bf170dbf2 100644
--- a/testing/mozbase/mozdevice/setup.py
+++ b/testing/mozbase/mozdevice/setup.py
@@ -5,7 +5,7 @@
 from setuptools import setup
 
 PACKAGE_NAME = "mozdevice"
-PACKAGE_VERSION = "4.1.1"
+PACKAGE_VERSION = "4.1.2"
 
 deps = ["mozlog >= 6.0"]
 
diff --git a/testing/mozbase/mozgeckoprofiler/mozgeckoprofiler/symbolication.py b/testing/mozbase/mozgeckoprofiler/mozgeckoprofiler/symbolication.py
index ecec5c1d9d..f7617a4e91 100644
--- a/testing/mozbase/mozgeckoprofiler/mozgeckoprofiler/symbolication.py
+++ b/testing/mozbase/mozgeckoprofiler/mozgeckoprofiler/symbolication.py
@@ -5,9 +5,9 @@ import hashlib
 import http.client
 import os
 import platform
+import shutil
 import subprocess
 import zipfile
-from distutils import spawn
 
 import six
 from mozlog import get_proxy_logger
@@ -95,7 +95,7 @@ class OSXSymbolDumper:
 
 class LinuxSymbolDumper:
     def __init__(self):
-        self.nm = spawn.find_executable("nm")
+        self.nm = shutil.which("nm")
         if not self.nm:
             raise SymbolError("Could not find nm, necessary for symbol dumping")
 
diff --git a/testing/mozbase/mozlog/mozlog/formatters/errorsummary.py b/testing/mozbase/mozlog/mozlog/formatters/errorsummary.py
index fcedb7ebd4..f03611d7da 100644
--- a/testing/mozbase/mozlog/mozlog/formatters/errorsummary.py
+++ b/testing/mozbase/mozlog/mozlog/formatters/errorsummary.py
@@ -46,6 +46,7 @@ class ErrorSummaryFormatter(BaseFormatter):
             "expected": item["expected"],
             "message": item.get("message"),
             "stack": item.get("stack"),
+            "modifiers": item.get("extra", {}).get("modifiers", ""),
             "known_intermittent": item.get("known_intermittent", []),
         }
         return self._output("test_result", data)
diff --git a/testing/mozbase/mozlog/tests/test_errorsummary.py b/testing/mozbase/mozlog/tests/test_errorsummary.py
index 30a5a304b2..0925c3a09e 100644
--- a/testing/mozbase/mozlog/tests/test_errorsummary.py
+++ b/testing/mozbase/mozlog/tests/test_errorsummary.py
@@ -33,7 +33,7 @@ import pytest
             ],
             """
                 {"groups": ["manifestA", "manifestB"], "action": "test_groups", "line": 0}
-                {"test": "test_baz", "subtest": null, "group": "manifestA", "status": "PASS", "expected": "FAIL", "message": null, "stack": null, "known_intermittent": [], "action": "test_result", "line": 8}
+                {"test": "test_baz", "subtest": null, "group": "manifestA", "status": "PASS", "expected": "FAIL", "message": null, "stack": null, "modifiers": "", "known_intermittent": [], "action": "test_result", "line": 8}
                 {"group": "manifestA", "status": "ERROR", "duration": 20, "action": "group_result", "line": 9}
                 {"group": "manifestB", "status": "OK", "duration": 10, "action": "group_result", "line": 9}
             """.strip(),
@@ -86,7 +86,7 @@ import pytest
             ],
             """
                 {"groups": ["manifestA", "manifestB"], "action": "test_groups", "line": 0}
-                {"test": "test_bar", "subtest": null, "group": "manifestA", "status": "CRASH", "expected": "OK", "message": null, "stack": null, "known_intermittent": [], "action": "test_result", "line": 4}
+                {"test": "test_bar", "subtest": null, "group": "manifestA", "status": "CRASH", "expected": "OK", "message": null, "stack": null, "modifiers": "", "known_intermittent": [], "action": "test_result", "line": 4}
                 {"group": "manifestA", "status": "ERROR", "duration": 20, "action": "group_result", "line": 9}
                 {"group": "manifestB", "status": "OK", "duration": 10, "action": "group_result", "line": 9}
             """.strip(),
@@ -143,8 +143,8 @@ import pytest
             ],
             """
                 {"groups": ["manifestA", "manifestB"], "action": "test_groups", "line": 0}
-                {"test": "test_baz", "subtest": "Test timed out", "group": "manifestA", "status": "FAIL", "expected": "PASS", "message": null, "stack": null, "known_intermittent": [], "action": "test_result", "line": 8}
-                {"test": "test_baz", "subtest": "", "group": "manifestA", "status": "TIMEOUT", "expected": "PASS", "message": null, "stack": null, "known_intermittent": [], "action": "test_result", "line": 9}
+                {"test": "test_baz", "subtest": "Test timed out", "group": "manifestA", "status": "FAIL", "expected": "PASS", "message": null, "stack": null, "modifiers": "", "known_intermittent": [], "action": "test_result", "line": 8}
+                {"test": "test_baz", "subtest": "", "group": "manifestA", "status": "TIMEOUT", "expected": "PASS", "message": null, "stack": null, "modifiers": "", "known_intermittent": [], "action": "test_result", "line": 9}
                 {"test": "manifestA", "group": "manifestA", "signature": "signature", "stackwalk_stderr": null, "stackwalk_stdout": null, "action": "crash", "line": 10}
                 {"group": "manifestA", "status": "ERROR", "duration": 49, "action": "group_result", "line": 12}
                 {"group": "manifestB", "status": "OK", "duration": 10, "action": "group_result", "line": 12}
@@ -225,7 +225,7 @@ import pytest
             ],
             """
                 {"groups": ["manifestA", "manifestB"], "action": "test_groups", "line": 0}
-                {"test": "test_baz", "group": "manifestA", "status": "FAIL", "expected": "OK", "subtest": null, "message": null, "stack": null, "known_intermittent": [], "action": "test_result", "line": 8}
+                {"test": "test_baz", "group": "manifestA", "status": "FAIL", "expected": "OK", "subtest": null, "message": null, "stack": null, "modifiers": "", "known_intermittent": [], "action": "test_result", "line": 8}
             """.strip(),
             id="timeout_no_group_status",
         ),
diff --git a/testing/mozbase/moznetwork/tests/test_moznetwork.py b/testing/mozbase/moznetwork/tests/test_moznetwork.py
index 5be960cca1..16b49ffd2e 100644
--- a/testing/mozbase/moznetwork/tests/test_moznetwork.py
+++ b/testing/mozbase/moznetwork/tests/test_moznetwork.py
@@ -4,8 +4,8 @@ Unit-Tests for moznetwork
 """
 
 import re
+import shutil
 import subprocess
-from distutils.spawn import find_executable
 from unittest import mock
 
 import mozinfo
@@ -37,7 +37,7 @@ def ip_addresses():
 
     cmd = None
     for command in commands:
-        if find_executable(command[0]):
+        if shutil.which(command[0]):
             cmd = command
             break
     else:
diff --git a/testing/mozbase/mozprofile/mozprofile/profile.py b/testing/mozbase/mozprofile/mozprofile/profile.py
index 25e10d23bf..0eef6c910f 100644
--- a/testing/mozbase/mozprofile/mozprofile/profile.py
+++ b/testing/mozbase/mozprofile/mozprofile/profile.py
@@ -196,7 +196,7 @@ class Profile(BaseProfile):
         locations=None,
         proxy=None,
         restore=True,
-        whitelistpaths=None,
+        allowlistpaths=None,
         **kwargs
     ):
         """
@@ -206,7 +206,7 @@ class Profile(BaseProfile):
         :param locations: ServerLocations object
         :param proxy: Setup a proxy
         :param restore: Flag for removing all custom settings during cleanup
-        :param whitelistpaths: List of paths to pass to Firefox to allow read
+        :param allowlistpaths: List of paths to pass to Firefox to allow read
             access to from the content process sandbox.
         """
         super(Profile, self).__init__(
@@ -219,7 +219,7 @@ class Profile(BaseProfile):
 
         self._locations = locations
         self._proxy = proxy
-        self._whitelistpaths = whitelistpaths
+        self._allowlistpaths = allowlistpaths
 
         # Initialize all class members
         self._reset()
@@ -249,30 +249,30 @@ class Profile(BaseProfile):
         self.permissions = Permissions(self._locations)
         prefs_js, user_js = self.permissions.network_prefs(self._proxy)
 
-        if self._whitelistpaths:
+        if self._allowlistpaths:
             # On macOS we don't want to support a generalized read whitelist,
             # and the macOS sandbox policy language doesn't have support for
             # lists, so we handle these specially.
             if platform.system() == "Darwin":
-                assert len(self._whitelistpaths) <= 2
-                if len(self._whitelistpaths) == 2:
+                assert len(self._allowlistpaths) <= 2
+                if len(self._allowlistpaths) == 2:
                     prefs_js.append(
                         (
                             "security.sandbox.content.mac.testing_read_path2",
-                            self._whitelistpaths[1],
+                            self._allowlistpaths[1],
                         )
                     )
                 prefs_js.append(
                     (
                         "security.sandbox.content.mac.testing_read_path1",
-                        self._whitelistpaths[0],
+                        self._allowlistpaths[0],
                     )
                 )
             else:
                 prefs_js.append(
                     (
                         "security.sandbox.content.read_path_whitelist",
-                        ",".join(self._whitelistpaths),
+                        ",".join(self._allowlistpaths),
                     )
                 )
         self.set_preferences(prefs_js, "prefs.js")
diff --git a/testing/mozbase/mozprofile/setup.py b/testing/mozbase/mozprofile/setup.py
index 8344bd83ae..12a6dad5cd 100644
--- a/testing/mozbase/mozprofile/setup.py
+++ b/testing/mozbase/mozprofile/setup.py
@@ -5,7 +5,7 @@
 from setuptools import setup
 
 PACKAGE_NAME = "mozprofile"
-PACKAGE_VERSION = "2.6.1"
+PACKAGE_VERSION = "3.0.0"
 
 deps = [
     "mozfile>=1.2",
diff --git a/testing/mozbase/mozrunner/mozrunner/devices/android_device.py b/testing/mozbase/mozrunner/mozrunner/devices/android_device.py
index 7cbec1b6f8..2db4743655 100644
--- a/testing/mozbase/mozrunner/mozrunner/devices/android_device.py
+++ b/testing/mozbase/mozrunner/mozrunner/devices/android_device.py
@@ -13,7 +13,6 @@ import subprocess
 import sys
 import telnetlib
 import time
-from distutils.spawn import find_executable
 from enum import Enum
 
 import six
@@ -942,7 +941,7 @@ def _find_sdk_exe(substs, exe, tools):
 
     if not found:
         # Is exe on PATH?
-        exe_path = find_executable(exe)
+        exe_path = shutil.which(exe)
         if exe_path:
             found = True
         else:
diff --git a/testing/mozbase/mozrunner/setup.py b/testing/mozbase/mozrunner/setup.py
index 72407ffd95..7233c17386 100644
--- a/testing/mozbase/mozrunner/setup.py
+++ b/testing/mozbase/mozrunner/setup.py
@@ -5,7 +5,7 @@
 from setuptools import find_packages, setup
 
 PACKAGE_NAME = "mozrunner"
-PACKAGE_VERSION = "8.3.0"
+PACKAGE_VERSION = "8.3.1"
 
 desc = """Reliable start/stop/configuration of Mozilla Applications (Firefox, Thunderbird, etc.)"""
 
@@ -15,7 +15,7 @@ deps = [
     "mozinfo>=0.7,<2",
     "mozlog>=6.0",
     "mozprocess>=1.3.0,<2",
-    "mozprofile~=2.3",
+    "mozprofile~=3.0",
     "six>=1.13.0,<2",
 ]
 
diff --git a/testing/mozbase/mozsystemmonitor/mozsystemmonitor/resourcemonitor.py b/testing/mozbase/mozsystemmonitor/mozsystemmonitor/resourcemonitor.py
index 978d7d6911..553fcde284 100644
--- a/testing/mozbase/mozsystemmonitor/mozsystemmonitor/resourcemonitor.py
+++ b/testing/mozbase/mozsystemmonitor/mozsystemmonitor/resourcemonitor.py
@@ -314,6 +314,7 @@ class SystemResourceMonitor(object):
         self._swap_type = type(swap)
         self._swap_len = len(swap)
         self.start_timestamp = time.time()
+        self.start_time = time.monotonic()
 
         self._pipe, child_pipe = multiprocessing.Pipe(True)
 
@@ -328,6 +329,9 @@ class SystemResourceMonitor(object):
             self._pipe.send(("terminate",))
             self._process.join()
 
+    def convert_to_monotonic_time(self, timestamp):
+        return timestamp - self.start_timestamp + self.start_time
+
     # Methods to control monitoring.
 
     def start(self):
@@ -458,18 +462,24 @@ class SystemResourceMonitor(object):
             SystemResourceMonitor.instance.markers.append((name, start, end, text))
 
     @staticmethod
-    def begin_marker(name, text, disambiguator=None):
+    def begin_marker(name, text, disambiguator=None, timestamp=None):
         if SystemResourceMonitor.instance:
             id = name + ":" + text
             if disambiguator:
                 id += ":" + disambiguator
-            SystemResourceMonitor.instance._active_markers[id] = time.monotonic()
+            SystemResourceMonitor.instance._active_markers[id] = (
+                SystemResourceMonitor.instance.convert_to_monotonic_time(timestamp)
+                if timestamp
+                else time.monotonic()
+            )
 
     @staticmethod
-    def end_marker(name, text, disambiguator=None):
+    def end_marker(name, text, disambiguator=None, timestamp=None):
         if not SystemResourceMonitor.instance:
             return
         end = time.monotonic()
+        if timestamp:
+            end = SystemResourceMonitor.instance.convert_to_monotonic_time(timestamp)
         id = name + ":" + text
         if disambiguator:
             id += ":" + disambiguator
diff --git a/testing/mozbase/moztest/moztest/resolve.py b/testing/mozbase/moztest/moztest/resolve.py
index 42bf0ebdda..d84efc22e2 100644
--- a/testing/mozbase/moztest/moztest/resolve.py
+++ b/testing/mozbase/moztest/moztest/resolve.py
@@ -14,7 +14,6 @@ import six
 from manifestparser import TestManifest, combine_fields
 from mozbuild.base import MozbuildObject
 from mozbuild.testing import REFTEST_FLAVORS, TEST_MANIFESTS
-from mozbuild.util import OrderedDefaultDict
 from mozpack.files import FileFinder
 
 here = os.path.abspath(os.path.dirname(__file__))
@@ -612,7 +611,7 @@ class TestResolver(MozbuildObject):
         self._wpt_loaded = False
 
     def _reset_state(self):
-        self._tests_by_path = OrderedDefaultDict(list)
+        self._tests_by_path = defaultdict(list)
         self._tests_by_flavor = defaultdict(set)
         self._tests_by_manifest = defaultdict(list)
         self._test_dirs = set()
@@ -887,7 +886,6 @@ class TestResolver(MozbuildObject):
             self.topsrcdir,
             self.topobjdir,
             rebuild=False,
-            download=True,
             config_path=None,
             rewrite_config=True,
             update=True,
diff --git a/testing/mozbase/rust/mozdevice/src/lib.rs b/testing/mozbase/rust/mozdevice/src/lib.rs
index 45f7d4d2cf..0bc6509e89 100644
--- a/testing/mozbase/rust/mozdevice/src/lib.rs
+++ b/testing/mozbase/rust/mozdevice/src/lib.rs
@@ -53,8 +53,6 @@ impl FromStr for AndroidStorageInput {
     }
 }
 
-
-
 #[derive(Debug, Clone, Copy, PartialEq)]
 pub enum AndroidStorage {
     App,
diff --git a/testing/mozbase/rust/mozversion/src/lib.rs b/testing/mozbase/rust/mozversion/src/lib.rs
index ccb6b01803..225b290d10 100644
--- a/testing/mozbase/rust/mozversion/src/lib.rs
+++ b/testing/mozbase/rust/mozversion/src/lib.rs
@@ -256,8 +256,8 @@ pub fn firefox_binary_version(binary: &Path) -> VersionResult<Version> {
 }
 
 fn parse_binary_version(version_str: &str) -> VersionResult<Version> {
-    let version_regexp = Regex::new(r#"Firefox[[:space:]]+(?P<version>.+)"#)
-        .expect("Error parsing version regexp");
+    let version_regexp =
+        Regex::new(r#"Firefox[[:space:]]+(?P<version>.+)"#).expect("Error parsing version regexp");
 
     let version_match = version_regexp
         .captures(version_str)