Adding upstream version 110.0.1.upstream/110.0.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 449 insertions, 0 deletions

diff --git a/testing/web-platform/tests/tools/manifest/manifest.py b/testing/web-platform/tests/tools/manifest/manifest.py
new file mode 100644
index 0000000000..4b7792ec00
--- /dev/null
+++ b/testing/web-platform/tests/tools/manifest/manifest.py
@@ -0,0 +1,449 @@
+import os
+import sys
+from atomicwrites import atomic_write
+from copy import deepcopy
+from multiprocessing import Pool, cpu_count
+
+from . import jsonlib
+from . import vcs
+from .item import (ConformanceCheckerTest,
+                   CrashTest,
+                   ManifestItem,
+                   ManualTest,
+                   PrintRefTest,
+                   RefTest,
+                   SupportFile,
+                   TestharnessTest,
+                   VisualTest,
+                   WebDriverSpecTest)
+from .log import get_logger
+from .sourcefile import SourceFile
+from .typedata import TypeData
+
+MYPY = False
+if MYPY:
+    # MYPY is set to True when run under Mypy.
+    from logging import Logger
+    from typing import Any
+    from typing import Container
+    from typing import Dict
+    from typing import IO
+    from typing import Iterator
+    from typing import Iterable
+    from typing import Optional
+    from typing import Set
+    from typing import Text
+    from typing import Tuple
+    from typing import Type
+    from typing import Union
+
+
+CURRENT_VERSION = 8  # type: int
+
+
+class ManifestError(Exception):
+    pass
+
+
+class ManifestVersionMismatch(ManifestError):
+    pass
+
+
+class InvalidCacheError(Exception):
+    pass
+
+
+item_classes = {"testharness": TestharnessTest,
+                "reftest": RefTest,
+                "print-reftest": PrintRefTest,
+                "crashtest": CrashTest,
+                "manual": ManualTest,
+                "wdspec": WebDriverSpecTest,
+                "conformancechecker": ConformanceCheckerTest,
+                "visual": VisualTest,
+                "support": SupportFile}  # type: Dict[Text, Type[ManifestItem]]
+
+
+def compute_manifest_items(source_file):
+    # type: (SourceFile) -> Tuple[Tuple[Text, ...], Text, Set[ManifestItem], Text]
+    rel_path_parts = source_file.rel_path_parts
+    new_type, manifest_items = source_file.manifest_items()
+    file_hash = source_file.hash
+    return rel_path_parts, new_type, set(manifest_items), file_hash
+
+
+if MYPY:
+    ManifestDataType = Dict[Any, TypeData]
+else:
+    ManifestDataType = dict
+
+
+class ManifestData(ManifestDataType):
+    def __init__(self, manifest):
+        # type: (Manifest) -> None
+        """Dictionary subclass containing a TypeData instance for each test type,
+        keyed by type name"""
+        self.initialized = False  # type: bool
+        for key, value in item_classes.items():
+            self[key] = TypeData(manifest, value)
+        self.initialized = True
+        self.json_obj = None  # type: None
+
+    def __setitem__(self, key, value):
+        # type: (Text, TypeData) -> None
+        if self.initialized:
+            raise AttributeError
+        dict.__setitem__(self, key, value)
+
+    def paths(self):
+        # type: () -> Set[Text]
+        """Get a list of all paths containing test items
+        without actually constructing all the items"""
+        rv = set()  # type: Set[Text]
+        for item_data in self.values():
+            for item in item_data:
+                rv.add(os.path.sep.join(item))
+        return rv
+
+    def type_by_path(self):
+        # type: () -> Dict[Tuple[Text, ...], Text]
+        rv = {}
+        for item_type, item_data in self.items():
+            for item in item_data:
+                rv[item] = item_type
+        return rv
+
+
+class Manifest:
+    def __init__(self, tests_root, url_base="/"):
+        # type: (Text, Text) -> None
+        assert url_base is not None
+        self._data = ManifestData(self)  # type: ManifestData
+        self.tests_root = tests_root  # type: Text
+        self.url_base = url_base  # type: Text
+
+    def __iter__(self):
+        # type: () -> Iterator[Tuple[Text, Text, Set[ManifestItem]]]
+        return self.itertypes()
+
+    def itertypes(self, *types):
+        # type: (*Text) -> Iterator[Tuple[Text, Text, Set[ManifestItem]]]
+        for item_type in (types or sorted(self._data.keys())):
+            for path in self._data[item_type]:
+                rel_path = os.sep.join(path)
+                tests = self._data[item_type][path]
+                yield item_type, rel_path, tests
+
+    def iterpath(self, path):
+        # type: (Text) -> Iterable[ManifestItem]
+        tpath = tuple(path.split(os.path.sep))
+
+        for type_tests in self._data.values():
+            i = type_tests.get(tpath, set())
+            assert i is not None
+            yield from i
+
+    def iterdir(self, dir_name):
+        # type: (Text) -> Iterable[ManifestItem]
+        tpath = tuple(dir_name.split(os.path.sep))
+        tpath_len = len(tpath)
+
+        for type_tests in self._data.values():
+            for path, tests in type_tests.items():
+                if path[:tpath_len] == tpath:
+                    yield from tests
+
+    def update(self, tree, parallel=True):
+        # type: (Iterable[Tuple[Text, Optional[Text], bool]], bool) -> bool
+        """Update the manifest given an iterable of items that make up the updated manifest.
+
+        The iterable must either generate tuples of the form (SourceFile, True) for paths
+        that are to be updated, or (path, False) for items that are not to be updated. This
+        unusual API is designed as an optimistaion meaning that SourceFile items need not be
+        constructed in the case we are not updating a path, but the absence of an item from
+        the iterator may be used to remove defunct entries from the manifest."""
+
+        logger = get_logger()
+
+        changed = False
+
+        # Create local variable references to these dicts so we avoid the
+        # attribute access in the hot loop below
+        data = self._data
+
+        types = data.type_by_path()
+        remaining_manifest_paths = set(types)
+
+        to_update = []
+
+        for path, file_hash, updated in tree:
+            path_parts = tuple(path.split(os.path.sep))
+            is_new = path_parts not in remaining_manifest_paths
+
+            if not updated and is_new:
+                # This is kind of a bandaid; if we ended up here the cache
+                # was invalid but we've been using it anyway. That's obviously
+                # bad; we should fix the underlying issue that we sometimes
+                # use an invalid cache. But at least this fixes the immediate
+                # problem
+                raise InvalidCacheError
+
+            if not updated:
+                remaining_manifest_paths.remove(path_parts)
+            else:
+                assert self.tests_root is not None
+                source_file = SourceFile(self.tests_root,
+                                         path,
+                                         self.url_base,
+                                         file_hash)
+
+                hash_changed = False  # type: bool
+
+                if not is_new:
+                    if file_hash is None:
+                        file_hash = source_file.hash
+                    remaining_manifest_paths.remove(path_parts)
+                    old_type = types[path_parts]
+                    old_hash = data[old_type].hashes[path_parts]
+                    if old_hash != file_hash:
+                        hash_changed = True
+                        del data[old_type][path_parts]
+
+                if is_new or hash_changed:
+                    to_update.append(source_file)
+
+        if to_update:
+            logger.debug("Computing manifest update for %s items" % len(to_update))
+            changed = True
+
+
+        # 25 items was derived experimentally (2020-01) to be approximately the
+        # point at which it is quicker to create a Pool and parallelize update.
+        pool = None
+        if parallel and len(to_update) > 25 and cpu_count() > 1:
+            # On Python 3 on Windows, using >= MAXIMUM_WAIT_OBJECTS processes
+            # causes a crash in the multiprocessing module. Whilst this enum
+            # can technically have any value, it is usually 64. For safety,
+            # restrict manifest regeneration to 48 processes on Windows.
+            #
+            # See https://bugs.python.org/issue26903 and https://bugs.python.org/issue40263
+            processes = cpu_count()
+            if sys.platform == "win32" and processes > 48:
+                processes = 48
+            pool = Pool(processes)
+
+            # chunksize set > 1 when more than 10000 tests, because
+            # chunking is a net-gain once we get to very large numbers
+            # of items (again, experimentally, 2020-01)
+            chunksize = max(1, len(to_update) // 10000)
+            logger.debug("Doing a multiprocessed update. CPU count: %s, "
+                "processes: %s, chunksize: %s" % (cpu_count(), processes, chunksize))
+            results = pool.imap_unordered(compute_manifest_items,
+                                          to_update,
+                                          chunksize=chunksize
+                                          )  # type: Iterator[Tuple[Tuple[Text, ...], Text, Set[ManifestItem], Text]]
+        else:
+            results = map(compute_manifest_items, to_update)
+
+        for result in results:
+            rel_path_parts, new_type, manifest_items, file_hash = result
+            data[new_type][rel_path_parts] = manifest_items
+            data[new_type].hashes[rel_path_parts] = file_hash
+
+        # Make sure to terminate the Pool, to avoid hangs on Python 3.
+        # https://docs.python.org/3/library/multiprocessing.html#multiprocessing.pool.Pool
+        if pool is not None:
+            pool.terminate()
+
+        if remaining_manifest_paths:
+            changed = True
+            for rel_path_parts in remaining_manifest_paths:
+                for test_data in data.values():
+                    if rel_path_parts in test_data:
+                        del test_data[rel_path_parts]
+
+        return changed
+
+    def to_json(self, caller_owns_obj=True):
+        # type: (bool) -> Dict[Text, Any]
+        """Dump a manifest into a object which can be serialized as JSON
+
+        If caller_owns_obj is False, then the return value remains
+        owned by the manifest; it is _vitally important_ that _no_
+        (even read) operation is done on the manifest, as otherwise
+        objects within the object graph rooted at the return value can
+        be mutated. This essentially makes this mode very dangerous
+        and only to be used under extreme care.
+
+        """
+        out_items = {
+            test_type: type_paths.to_json()
+            for test_type, type_paths in self._data.items() if type_paths
+        }
+
+        if caller_owns_obj:
+            out_items = deepcopy(out_items)
+
+        rv = {"url_base": self.url_base,
+              "items": out_items,
+              "version": CURRENT_VERSION}  # type: Dict[Text, Any]
+        return rv
+
+    @classmethod
+    def from_json(cls, tests_root, obj, types=None, callee_owns_obj=False):
+        # type: (Text, Dict[Text, Any], Optional[Container[Text]], bool) -> Manifest
+        """Load a manifest from a JSON object
+
+        This loads a manifest for a given local test_root path from an
+        object obj, potentially partially loading it to only load the
+        types given by types.
+
+        If callee_owns_obj is True, then ownership of obj transfers
+        to this function when called, and the caller must never mutate
+        the obj or anything referred to in the object graph rooted at
+        obj.
+
+        """
+        version = obj.get("version")
+        if version != CURRENT_VERSION:
+            raise ManifestVersionMismatch
+
+        self = cls(tests_root, url_base=obj.get("url_base", "/"))
+        if not hasattr(obj, "items"):
+            raise ManifestError
+
+        for test_type, type_paths in obj["items"].items():
+            if test_type not in item_classes:
+                raise ManifestError
+
+            if types and test_type not in types:
+                continue
+
+            if not callee_owns_obj:
+                type_paths = deepcopy(type_paths)
+
+            self._data[test_type].set_json(type_paths)
+
+        return self
+
+
+def load(tests_root, manifest, types=None):
+    # type: (Text, Union[IO[bytes], Text], Optional[Container[Text]]) -> Optional[Manifest]
+    logger = get_logger()
+
+    logger.warning("Prefer load_and_update instead")
+    return _load(logger, tests_root, manifest, types)
+
+
+__load_cache = {}  # type: Dict[Text, Manifest]
+
+
+def _load(logger,  # type: Logger
+          tests_root,  # type: Text
+          manifest,  # type: Union[IO[bytes], Text]
+          types=None,  # type: Optional[Container[Text]]
+          allow_cached=True  # type: bool
+          ):
+    # type: (...) -> Optional[Manifest]
+    manifest_path = (manifest if isinstance(manifest, str)
+                     else manifest.name)
+    if allow_cached and manifest_path in __load_cache:
+        return __load_cache[manifest_path]
+
+    if isinstance(manifest, str):
+        if os.path.exists(manifest):
+            logger.debug("Opening manifest at %s" % manifest)
+        else:
+            logger.debug("Creating new manifest at %s" % manifest)
+        try:
+            with open(manifest, encoding="utf-8") as f:
+                rv = Manifest.from_json(tests_root,
+                                        jsonlib.load(f),
+                                        types=types,
+                                        callee_owns_obj=True)
+        except OSError:
+            return None
+        except ValueError:
+            logger.warning("%r may be corrupted", manifest)
+            return None
+    else:
+        rv = Manifest.from_json(tests_root,
+                                jsonlib.load(manifest),
+                                types=types,
+                                callee_owns_obj=True)
+
+    if allow_cached:
+        __load_cache[manifest_path] = rv
+    return rv
+
+
+def load_and_update(tests_root,  # type: Text
+                    manifest_path,  # type: Text
+                    url_base,  # type: Text
+                    update=True,  # type: bool
+                    rebuild=False,  # type: bool
+                    metadata_path=None,  # type: Optional[Text]
+                    cache_root=None,  # type: Optional[Text]
+                    working_copy=True,  # type: bool
+                    types=None,  # type: Optional[Container[Text]]
+                    write_manifest=True,  # type: bool
+                    allow_cached=True,  # type: bool
+                    parallel=True  # type: bool
+                    ):
+    # type: (...) -> Manifest
+
+    logger = get_logger()
+
+    manifest = None
+    if not rebuild:
+        try:
+            manifest = _load(logger,
+                             tests_root,
+                             manifest_path,
+                             types=types,
+                             allow_cached=allow_cached)
+        except ManifestVersionMismatch:
+            logger.info("Manifest version changed, rebuilding")
+        except ManifestError:
+            logger.warning("Failed to load manifest, rebuilding")
+
+        if manifest is not None and manifest.url_base != url_base:
+            logger.info("Manifest url base did not match, rebuilding")
+            manifest = None
+
+    if manifest is None:
+        manifest = Manifest(tests_root, url_base)
+        rebuild = True
+        update = True
+
+    if rebuild or update:
+        logger.info("Updating manifest")
+        for retry in range(2):
+            try:
+                tree = vcs.get_tree(tests_root, manifest, manifest_path, cache_root,
+                                    working_copy, rebuild)
+                changed = manifest.update(tree, parallel)
+                break
+            except InvalidCacheError:
+                logger.warning("Manifest cache was invalid, doing a complete rebuild")
+                rebuild = True
+        else:
+            # If we didn't break there was an error
+            raise
+        if write_manifest and changed:
+            write(manifest, manifest_path)
+        tree.dump_caches()
+
+    return manifest
+
+
+def write(manifest, manifest_path):
+    # type: (Manifest, Text) -> None
+    dir_name = os.path.dirname(manifest_path)
+    if not os.path.exists(dir_name):
+        os.makedirs(dir_name)
+    with atomic_write(manifest_path, overwrite=True) as f:
+        # Use ',' instead of the default ', ' separator to prevent trailing
+        # spaces: https://docs.python.org/2/library/json.html#json.dump
+        jsonlib.dump_dist(manifest.to_json(caller_owns_obj=True), f)
+        f.write("\n")