path: root/testing/web-platform/tests/html/canvas/tools/gentestutilsunion.py

"""Generates Canvas tests from YAML file definitions."""
# Current code status:
#
# This was originally written by Philip Taylor for use at
# http://philip.html5.org/tests/canvas/suite/tests/
#
# It has been adapted for use with the Web Platform Test Suite suite at
# https://github.com/web-platform-tests/wpt/
#
# The original version had a number of now-removed features (multiple versions
# of each test case of varying verbosity, Mozilla mochitests, semi-automated
# test harness). It also had a different directory structure.

# To update or add test cases:
#
# * Modify the tests*.yaml files.
#  - 'name' is an arbitrary hierarchical name to help categorise tests.
#  - 'desc' is a rough description of what behaviour the test aims to test.
#  - 'code' is JavaScript code to execute, with some special commands starting
#    with '@'.
#  - 'expected' is what the final canvas output should be: a string 'green' or
#    'clear' (100x50 images in both cases), or a string 'size 100 50' (or any
#    other size) followed by Python code using Pycairo to generate the image.
#
# * Run "./build.sh".
# This requires a few Python modules which might not be ubiquitous.
# It will usually emit some warnings, which ideally should be fixed but can
# generally be safely ignored.
#
# * Test the tests, add new ones to Git, remove deleted ones from Git, etc.

from typing import Any, DefaultDict, FrozenSet, List, Mapping, MutableMapping
from typing import Optional, Set, Tuple

import re
import collections
import dataclasses
import enum
import importlib
import itertools
import os
import pathlib
import sys
import textwrap

import jinja2

try:
    import cairocffi as cairo  # type: ignore
except ImportError:
    import cairo

try:
    # Compatible and lots faster.
    import syck as yaml  # type: ignore
except ImportError:
    import yaml


class Error(Exception):
    """Base class for all exceptions raised by this module"""


class InvalidTestDefinitionError(Error):
    """Raised on invalid test definition."""


def _double_quote_escape(string: str) -> str:
    return string.replace('\\', '\\\\').replace('"', '\\"')


def _escape_js(string: str) -> str:
    string = _double_quote_escape(string)
    # Kind of an ugly hack, for nicer failure-message output.
    string = re.sub(r'\[(\w+)\]', r'[\\""+(\1)+"\\"]', string)
    return string


def _unroll(text: str) -> str:
    """Unrolls text with all possible permutations of the parameter lists.

    Example:
    >>> print _unroll('f = {<a | b>: <1 | 2 | 3>};')
    // a
    f = {a: 1};
    f = {a: 2};
    f = {a: 3};
    // b
    f = {b: 1};
    f = {b: 2};
    f = {b: 3};
    """
    patterns = []  # type: List[Tuple[str, List[str]]]
    while True:
        match = re.search(r'<([^>]+)>', text)
        if not match:
            break
        key = f'@unroll_pattern_{len(patterns)}'
        values = text[match.start(1):match.end(1)]
        text = text[:match.start(0)] + key + text[match.end(0):]
        patterns.append((key, [value.strip() for value in values.split('|')]))

    def unroll_patterns(text: str,
                        patterns: List[Tuple[str, List[str]]],
                        label: Optional[str] = None) -> List[str]:
        if not patterns:
            return [text]
        patterns = patterns.copy()
        key, values = patterns.pop(0)
        return (['// ' + label] if label else []) + list(
            itertools.chain.from_iterable(
                unroll_patterns(text.replace(key, value), patterns, value)
                for value in values))

    result = '\n'.join(unroll_patterns(text, patterns))
    return result


def _expand_nonfinite(method: str, argstr: str, tail: str) -> str:
    """
    >>> print _expand_nonfinite('f', '<0 a>, <0 b>', ';')
    f(a, 0);
    f(0, b);
    f(a, b);
    >>> print _expand_nonfinite('f', '<0 a>, <0 b c>, <0 d>', ';')
    f(a, 0, 0);
    f(0, b, 0);
    f(0, c, 0);
    f(0, 0, d);
    f(a, b, 0);
    f(a, b, d);
    f(a, 0, d);
    f(0, b, d);
    """
    # argstr is "<valid-1 invalid1-1 invalid2-1 ...>, ..." (where usually
    # 'invalid' is Infinity/-Infinity/NaN).
    args = []
    for arg in argstr.split(', '):
        match = re.match('<(.*)>', arg)
        if match is None:
            raise InvalidTestDefinitionError(
                f'Expected arg to match format "<(.*)>", but was: {arg}')
        a = match.group(1)
        args.append(a.split(' '))
    calls = []
    # Start with the valid argument list.
    call = [args[j][0] for j in range(len(args))]
    # For each argument alone, try setting it to all its invalid values:
    for i, arg in enumerate(args):
        for a in arg[1:]:
            c2 = call[:]
            c2[i] = a
            calls.append(c2)
    # For all combinations of >= 2 arguments, try setting them to their
    # first invalid values. (Don't do all invalid values, because the
    # number of combinations explodes.)
    def f(c: List[str], start: int, depth: int) -> None:
        for i in range(start, len(args)):
            if len(args[i]) > 1:
                a = args[i][1]
                c2 = c[:]
                c2[i] = a
                if depth > 0:
                    calls.append(c2)
                f(c2, i + 1, depth + 1)

    f(call, 0, 0)

    str_calls = (', '.join(c) for c in calls)
    return '\n'.join(f'{method}({params}){tail}' for params in str_calls)


def _get_test_sub_dir(name: str, name_to_sub_dir: Mapping[str, str]) -> str:
    for prefix in sorted(name_to_sub_dir.keys(), key=len, reverse=True):
        if name.startswith(prefix):
            return name_to_sub_dir[prefix]
    raise InvalidTestDefinitionError(
        f'Test "{name}" has no defined target directory mapping')


def _remove_extra_newlines(text: str) -> str:
    """Remove newlines if a backslash is found at end of line."""
    # Lines ending with '\' gets their newline character removed.
    text = re.sub(r'\\\n', '', text, flags=re.MULTILINE | re.DOTALL)

    # Lines ending with '\-' gets their newline and any leading white spaces on
    # the following line removed.
    text = re.sub(r'\\-\n\s*', '', text, flags=re.MULTILINE | re.DOTALL)
    return text


def _expand_test_code(code: str) -> str:
    code = re.sub(r' @moz-todo', '', code)

    code = re.sub(r'@moz-UniversalBrowserRead;', '', code)

    code = _remove_extra_newlines(code)

    # Unroll expressions with a cross-product-style parameter expansion.
    code = re.sub(r'@unroll ([^;]*;)', lambda m: _unroll(m.group(1)), code)

    code = re.sub(r'@nonfinite ([^(]+)\(([^)]+)\)(.*)', lambda m:
                  _expand_nonfinite(m.group(1), m.group(2), m.group(3)),
                  code)  # Must come before '@assert throws'.

    code = re.sub(r'@assert pixel (\d+,\d+) == (\d+,\d+,\d+,\d+);',
                  r'_assertPixel(canvas, \1, \2);', code)

    code = re.sub(r'@assert pixel (\d+,\d+) ==~ (\d+,\d+,\d+,\d+);',
                  r'_assertPixelApprox(canvas, \1, \2, 2);', code)

    code = re.sub(r'@assert pixel (\d+,\d+) ==~ (\d+,\d+,\d+,\d+) \+/- (\d+);',
                  r'_assertPixelApprox(canvas, \1, \2, \3);', code)

    code = re.sub(r'@assert throws (\S+_ERR) (.*?);$',
                  r'assert_throws_dom("\1", function() { \2; });', code,
                  flags=re.MULTILINE | re.DOTALL)

    code = re.sub(r'@assert throws (\S+Error) (.*?);$',
                  r'assert_throws_js(\1, function() { \2; });', code,
                  flags=re.MULTILINE | re.DOTALL)

    code = re.sub(
        r'@assert (.*) === (.*);', lambda m:
        (f'_assertSame({m.group(1)}, {m.group(2)}, '
         f'"{_escape_js(m.group(1))}", "{_escape_js(m.group(2))}");'), code)

    code = re.sub(
        r'@assert (.*) !== (.*);', lambda m:
        (f'_assertDifferent({m.group(1)}, {m.group(2)}, '
         f'"{_escape_js(m.group(1))}", "{_escape_js(m.group(2))}");'), code)

    code = re.sub(
        r'@assert (.*) =~ (.*);',
        lambda m: f'assert_regexp_match({m.group(1)}, {m.group(2)});', code)

    code = re.sub(
        r'@assert (.*);',
        lambda m: f'_assert({m.group(1)}, "{_escape_js(m.group(1))}");', code)

    assert '@' not in code

    return code


_TestParams = Mapping[str, Any]
_MutableTestParams = MutableMapping[str, Any]


class _CanvasType(str, enum.Enum):
    HTML_CANVAS = 'HtmlCanvas'
    OFFSCREEN_CANVAS = 'OffscreenCanvas'
    WORKER = 'Worker'


class _TemplateType(str, enum.Enum):
    REFERENCE = 'reference'
    HTML_REFERENCE = 'html_reference'
    TESTHARNESS = 'testharness'


@dataclasses.dataclass
class _OutputPaths:
    element: str
    offscreen: str

    def sub_path(self, sub_dir: str):
        """Create a new _OutputPaths that is a subpath of this _OutputPath."""
        return _OutputPaths(element=os.path.join(self.element, sub_dir),
                            offscreen=os.path.join(self.offscreen, sub_dir))


def _validate_test(test: _TestParams):
    if test.get('expected', '') == 'green' and re.search(
            r'@assert pixel .* 0,0,0,0;', test['code']):
        print(f'Probable incorrect pixel test in {test["name"]}')

    if 'size' in test and (not isinstance(test['size'], list)
                           or len(test['size']) != 2):
        raise InvalidTestDefinitionError(
            f'Invalid canvas size "{test["size"]}" in test {test["name"]}. '
            'Expected an array with two numbers.')

    if test['template_type'] == _TemplateType.TESTHARNESS:
        valid_test_types = {'sync', 'async', 'promise'}
    else:
        valid_test_types = {'promise'}

    test_type = test.get('test_type')
    if test_type is not None and test_type not in valid_test_types:
        raise InvalidTestDefinitionError(
            f'Invalid test_type: {test_type}. '
            f'Valid values are: {valid_test_types}.')


def _render_template(jinja_env: jinja2.Environment, template: jinja2.Template,
                     params: _TestParams) -> str:
    """Renders the specified jinja template.

    The template is repetitively rendered until no more changes are observed.
    This allows for template parameters to refer to other template parameters.
    """
    rendered = template.render(params)
    previous = ''
    while rendered != previous:
        previous = rendered
        template = jinja_env.from_string(rendered)
        rendered = template.render(params)
    return rendered


def _render(jinja_env: jinja2.Environment, template_name: str,
            params: _TestParams, output_file_name: str):
    template = jinja_env.get_template(template_name)
    file_content = _render_template(jinja_env, template, params)
    pathlib.Path(output_file_name).write_text(file_content, 'utf-8')


def _preprocess_code(jinja_env: jinja2.Environment, code: str,
                     params: _TestParams) -> str:
    code = _expand_test_code(code)
    # Render the code on its own, as it could contain templates expanding
    # to multiple lines. This is needed to get proper indentation of the
    # code in the main template.
    code = _render_template(jinja_env, jinja_env.from_string(code), params)
    return code


class _Variant():

    def __init__(self, params: _MutableTestParams) -> None:
        self._params = params

    @property
    def params(self) -> _TestParams:
        """Read-only getter for this variant's param dict."""
        return self._params

    @staticmethod
    def create_with_defaults(test: _TestParams) -> '_Variant':
        """Create a _Variant from the specified params.

        Default values are added for certain parameters, if missing."""
        params = {
            'desc': '',
            'size': [100, 50],
            'variant_names': [],
        }
        params.update(test)
        return _Variant(params)

    def _get_variant_name(self, jinja_env: jinja2.Environment) -> str:
        name = self.params['name']
        if self.params.get('append_variants_to_name', True):
            name = '.'.join([name] + self.params['variant_names'])

        name = jinja_env.from_string(name).render(self.params)
        return name

    def _get_file_name(self) -> str:
        file_name = self.params['name']

        if 'manual' in self.params:
            file_name += '-manual'

        return file_name

    def _get_canvas_types(self) -> FrozenSet[_CanvasType]:
        canvas_types = self.params.get('canvas_types', _CanvasType)
        invalid_types = {
            type
            for type in canvas_types if type not in list(_CanvasType)
        }
        if invalid_types:
            raise InvalidTestDefinitionError(
                f'Invalid canvas_types: {list(invalid_types)}. '
                f'Accepted values are: {[t.value for t in _CanvasType]}')
        return frozenset(_CanvasType(t) for t in canvas_types)

    def _get_template_type(self) -> _TemplateType:
        if 'reference' in self.params and 'html_reference' in self.params:
            raise InvalidTestDefinitionError(
                f'Test {self.params["name"]} is invalid, "reference" and '
                '"html_reference" can\'t both be specified at the same time.')

        if 'reference' in self.params:
            return _TemplateType.REFERENCE
        if 'html_reference' in self.params:
            return _TemplateType.HTML_REFERENCE
        return _TemplateType.TESTHARNESS

    def finalize_params(self, jinja_env: jinja2.Environment) -> None:
        """Finalize this variant by adding computed param fields."""
        self._params['name'] = self._get_variant_name(jinja_env)
        self._params['file_name'] = self._get_file_name()
        self._params['canvas_types'] = self._get_canvas_types()
        self._params['template_type'] = self._get_template_type()

        if 'reference' in self._params:
            self._params['reference'] = _preprocess_code(
                jinja_env, self._params['reference'], self._params)

        if 'html_reference' in self._params:
            self._params['html_reference'] = _preprocess_code(
                jinja_env, self._params['html_reference'], self._params)

        code_params = dict(self.params)
        if _CanvasType.HTML_CANVAS in self.params['canvas_types']:
            code_params['canvas_type'] = _CanvasType.HTML_CANVAS.value
            self._params['code_element'] = _preprocess_code(
                jinja_env, self._params['code'], code_params)

        if _CanvasType.OFFSCREEN_CANVAS in self.params['canvas_types']:
            code_params['canvas_type'] = _CanvasType.OFFSCREEN_CANVAS.value
            self._params['code_offscreen'] = _preprocess_code(
                jinja_env, self._params['code'], code_params)

        if _CanvasType.WORKER in self.params['canvas_types']:
            code_params['canvas_type'] = _CanvasType.WORKER.value
            self._params['code_worker'] = _preprocess_code(
                jinja_env, self._params['code'], code_params)

        _validate_test(self._params)

    def generate_expected_image(self, output_dirs: _OutputPaths) -> None:
        """Creates a reference image using Cairo and save filename in params."""
        if 'expected' not in self.params:
            return

        expected = self.params['expected']
        name = self.params['name']

        if expected == 'green':
            self._params['expected_img'] = '/images/green-100x50.png'
            return
        if expected == 'clear':
            self._params['expected_img'] = '/images/clear-100x50.png'
            return
        if ';' in expected:
            print(f'Found semicolon in {name}')
        expected = re.sub(
            r'^size (\d+) (\d+)',
            r'surface = cairo.ImageSurface(cairo.FORMAT_ARGB32, \1, \2)'
            r'\ncr = cairo.Context(surface)', expected)

        output_paths = output_dirs.sub_path(name)
        if _CanvasType.HTML_CANVAS in self.params['canvas_types']:
            expected_canvas = (
                f'{expected}\n'
                f'surface.write_to_png("{output_paths.element}.png")\n')
            eval(compile(expected_canvas, f'<test {name}>', 'exec'), {},
                 {'cairo': cairo})

        if {_CanvasType.OFFSCREEN_CANVAS, _CanvasType.WORKER
            } & self.params['canvas_types']:
            expected_offscreen = (
                f'{expected}\n'
                f'surface.write_to_png("{output_paths.offscreen}.png")\n')
            eval(compile(expected_offscreen, f'<test {name}>', 'exec'), {},
                 {'cairo': cairo})

        self._params['expected_img'] = f'{name}.png'

    def _write_reference_test(self, jinja_env: jinja2.Environment,
                              output_files: _OutputPaths):
        params = dict(self.params)
        if _CanvasType.HTML_CANVAS in params['canvas_types']:
            _render(jinja_env, 'reftest_element.html', params,
                    f'{output_files.element}.html')
        if _CanvasType.OFFSCREEN_CANVAS in params['canvas_types']:
            _render(jinja_env, 'reftest_offscreen.html', params,
                    f'{output_files.offscreen}.html')
        if _CanvasType.WORKER in params['canvas_types']:
            _render(jinja_env, 'reftest_worker.html', params,
                    f'{output_files.offscreen}.w.html')

        params['is_test_reference'] = True
        is_html_ref = params['template_type'] == _TemplateType.HTML_REFERENCE
        ref_template = 'reftest.html' if is_html_ref else 'reftest_element.html'
        if _CanvasType.HTML_CANVAS in params['canvas_types']:
            _render(jinja_env, ref_template, params,
                    f'{output_files.element}-expected.html')
        if {_CanvasType.OFFSCREEN_CANVAS, _CanvasType.WORKER
            } & params['canvas_types']:
            _render(jinja_env, ref_template, params,
                    f'{output_files.offscreen}-expected.html')

    def _write_testharness_test(self, jinja_env: jinja2.Environment,
                                output_files: _OutputPaths):
        # Create test cases for canvas and offscreencanvas.
        if _CanvasType.HTML_CANVAS in self.params['canvas_types']:
            _render(jinja_env, 'testharness_element.html', self.params,
                    f'{output_files.element}.html')

        if _CanvasType.OFFSCREEN_CANVAS in self.params['canvas_types']:
            _render(jinja_env, 'testharness_offscreen.html', self.params,
                    f'{output_files.offscreen}.html')

        if _CanvasType.WORKER in self.params['canvas_types']:
            _render(jinja_env, 'testharness_worker.js', self.params,
                    f'{output_files.offscreen}.worker.js')

    def generate_test(self, jinja_env: jinja2.Environment,
                      output_dirs: _OutputPaths) -> None:
        """Generate the test files to the specified output dirs."""
        output_files = output_dirs.sub_path(self.params['file_name'])

        if self.params['template_type'] in (_TemplateType.REFERENCE,
                                            _TemplateType.HTML_REFERENCE):
            self._write_reference_test(jinja_env, output_files)
        else:
            self._write_testharness_test(jinja_env, output_files)


def _recursive_expand_variant_matrix(original_test: _TestParams,
                                     variant_matrix: List[_TestParams],
                                     current_selection: List[Tuple[str, Any]],
                                     test_variants: List[_Variant]):
    if len(current_selection) == len(variant_matrix):
        # Selection for each variant is done, so add a new test to test_list.
        test = dict(original_test)
        variant_name_list = []
        for variant_name, variant_params in current_selection:
            test.update(variant_params)
            variant_name_list.append(variant_name)
        # Expose variant names as a list so they can be used from the yaml
        # files, which helps with better naming of tests.
        test.update({'variant_names': variant_name_list})
        test_variants.append(_Variant.create_with_defaults(test))
    else:
        # Continue the recursion with each possible selection for the current
        # variant.
        variant = variant_matrix[len(current_selection)]
        for variant_options in variant.items():
            current_selection.append(variant_options)
            _recursive_expand_variant_matrix(original_test, variant_matrix,
                                             current_selection, test_variants)
            current_selection.pop()


def _get_variants(test: _TestParams) -> List[_Variant]:
    current_selection = []
    test_variants = []
    variants = test.get('variants', [])
    if not isinstance(variants, list):
        raise InvalidTestDefinitionError(
            textwrap.dedent("""
            Variants must be specified as a list of variant dimensions, e.g.:
              variants:
              - dimension1-variant1:
                  param: ...
                dimension1-variant2:
                  param: ...
              - dimension2-variant1:
                  param: ...
                dimension2-variant2:
                  param: ..."""))
    _recursive_expand_variant_matrix(test, variants, current_selection,
                                     test_variants)
    return test_variants


def _check_uniqueness(tested: DefaultDict[str, Set[_CanvasType]], name: str,
                      canvas_types: FrozenSet[_CanvasType]) -> None:
    already_tested = tested[name].intersection(canvas_types)
    if already_tested:
        raise InvalidTestDefinitionError(
            f'Test {name} is defined twice for types {already_tested}')
    tested[name].update(canvas_types)


def generate_test_files(name_to_dir_file: str) -> None:
    """Generate Canvas tests from YAML file definition."""
    output_dirs = _OutputPaths(element='../element', offscreen='../offscreen')

    jinja_env = jinja2.Environment(
        loader=jinja2.PackageLoader('gentestutilsunion'),
        keep_trailing_newline=True,
        trim_blocks=True,
        lstrip_blocks=True)

    jinja_env.filters['double_quote_escape'] = _double_quote_escape

    # Run with --test argument to run unit tests.
    if len(sys.argv) > 1 and sys.argv[1] == '--test':
        doctest = importlib.import_module('doctest')
        doctest.testmod()
        sys.exit()

    name_to_sub_dir = (yaml.safe_load(
        pathlib.Path(name_to_dir_file).read_text(encoding='utf-8')))

    tests = []
    test_yaml_directory = 'yaml-new'
    yaml_files = [
        os.path.join(test_yaml_directory, f)
        for f in os.listdir(test_yaml_directory) if f.endswith('.yaml')
    ]
    for t in sum([
            yaml.safe_load(pathlib.Path(f).read_text(encoding='utf-8'))
            for f in yaml_files
    ], []):
        if 'DISABLED' in t:
            continue
        if 'meta' in t:
            eval(compile(t['meta'], '<meta test>', 'exec'), {},
                 {'tests': tests})
        else:
            tests.append(t)

    # Ensure the test output directories exist.
    test_dirs = [output_dirs.element, output_dirs.offscreen]
    for sub_dir in set(name_to_sub_dir.values()):
        test_dirs.append(f'{output_dirs.element}/{sub_dir}')
        test_dirs.append(f'{output_dirs.offscreen}/{sub_dir}')
    for d in test_dirs:
        try:
            os.mkdir(d)
        except FileExistsError:
            pass  # Ignore if it already exists,

    used_tests = collections.defaultdict(set)
    for test in tests:
        print(test['name'])
        for variant in _get_variants(test):
            variant.finalize_params(jinja_env)
            if test['name'] != variant.params['name']:
                print(f'  {variant.params["name"]}')

            sub_dir = _get_test_sub_dir(variant.params['file_name'],
                                        name_to_sub_dir)
            output_sub_dirs = output_dirs.sub_path(sub_dir)

            _check_uniqueness(used_tests, variant.params['name'],
                              variant.params['canvas_types'])
            variant.generate_expected_image(output_sub_dirs)
            variant.generate_test(jinja_env, output_sub_dirs)

    print()