diff options
Diffstat (limited to 'share/extensions/inkex/tester/__init__.py')
| -rw-r--r-- | share/extensions/inkex/tester/__init__.py | 384 |
1 files changed, 384 insertions, 0 deletions
diff --git a/share/extensions/inkex/tester/__init__.py b/share/extensions/inkex/tester/__init__.py new file mode 100644 index 0000000..f33f695 --- /dev/null +++ b/share/extensions/inkex/tester/__init__.py @@ -0,0 +1,384 @@ +# coding=utf-8 +# +# Copyright (C) 2018-2019 Martin Owens +# 2019 Thomas Holder +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110, USA. +# +""" +All Inkscape extensions should come with tests. This package provides you with +the tools needed to create tests and thus ensure that your extension continues +to work with future versions of Inkscape, the "inkex" python modules, and other +python and non-python tools you may use. + +Make sure your extension is a python extension and is using the `inkex.generic` +base classes. These provide the greatest amount of functionality for testing. + +You should start by creating a folder in your repository called `tests` with +an empty file inside called `__init__.py` to turn it into a module folder. + +For each of your extensions, you should create a file called +`test_{extension_name}.py` where the name reflects the name of your extension. + +There are two types of tests: + + 1. Full-process Comparison tests - These are tests which invoke your + extension with various arguments and attempt to compare the + output to a known good reference. These are useful for testing + that your extension would work if it was used in Inkscape. + + Good example of writing comparison tests can be found in the + Inkscape core repository, each test which inherits from + the ComparisonMixin class is running comparison tests. + + 2. Unit tests - These are individual test functions which call out to + specific functions within your extension. These are typical + python unit tests and many good python documents exist + to describe how to write them well. For examples here you + can find the tests that test the inkex modules themselves + to be the most instructive. + +When running a test, it will cause a certain fraction of the code within the +extension to execute. This fraction called it's **coverage** and a higher +coverage score indicates that your test is better at exercising the various +options, features, and branches within your code. + +Generating comparison output can be done using the EXPORT_COMPARE environment +variable when calling pytest. For example: + + EXPORT_COMPARE=1 pytest tests/test_my_specific_test.py + +This will create files in `tests/data/refs/*.out.export` and these files should +be manually checked to make sure they are correct before being renamed and stripped +of the `.export` suffix. pytest should then be re-run to confirm before +committing to the repository. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +import os +import re +import sys +import shutil +import tempfile +import hashlib +import random +import uuid + +from io import BytesIO, StringIO +import xml.etree.ElementTree as xml + +from unittest import TestCase as BaseCase +from inkex.base import InkscapeExtension + +from ..utils import PY3, to_bytes +from .xmldiff import xmldiff +from .mock import MockCommandMixin, Capture + +if False: # pylint: disable=using-constant-test + from typing import Type, List + from .filters import Compare + + +class NoExtension(InkscapeExtension): # pylint: disable=too-few-public-methods + """Test case must specify 'self.effect_class' to assertEffect.""" + + def __init__(self, *args, **kwargs): # pylint: disable=super-init-not-called + raise NotImplementedError(self.__doc__) + + def run(self, args=None, output=None): + """Fake run""" + pass + + +class TestCase(MockCommandMixin, BaseCase): + """ + Base class for all effects tests, provides access to data_files and test_without_parameters + """ + effect_class = NoExtension # type: Type[InkscapeExtension] + effect_name = property(lambda self: self.effect_class.__module__) + + # If set to true, the output is not expected to be the stdout SVG document, but rather + # text or a message sent to the stderr, this is highly weird. But sometimes happens. + stderr_output = False + stdout_protect = True + stderr_protect = True + python3_only = False + + def __init__(self, *args, **kw): + super(TestCase, self).__init__(*args, **kw) + self._temp_dir = None + self._effect = None + + def setUp(self): # pylint: disable=invalid-name + """Make sure every test is seeded the same way""" + self._effect = None + super(TestCase, self).setUp() + if self.python3_only and not PY3: + self.skipTest("No available in python2") + try: + # python3, with version 1 to get the same numbers + # as in python2 during tests. + random.seed(0x35f, version=1) + except TypeError: + # But of course this kwarg doesn't exist in python2 + random.seed(0x35f) + + def tearDown(self): + super(TestCase, self).tearDown() + if self._temp_dir and os.path.isdir(self._temp_dir): + shutil.rmtree(self._temp_dir) + + @classmethod + def __file__(cls): + """Create a __file__ property which acts much like the module version""" + return os.path.abspath(sys.modules[cls.__module__].__file__) + + @classmethod + def _testdir(cls): + """Get's the folder where the test exists (so data can be found)""" + return os.path.dirname(cls.__file__()) + + @classmethod + def rootdir(cls): + """Return the full path to the extensions directory""" + return os.path.dirname(cls._testdir()) + + @classmethod + def datadir(cls): + """Get the data directory (can be over-ridden if needed)""" + return os.path.join(cls._testdir(), 'data') + + @property + def tempdir(self): + """Generate a temporary location to store files""" + if self._temp_dir is None: + self._temp_dir = tempfile.mkdtemp(prefix='inkex-tests-') + if not os.path.isdir(self._temp_dir): + raise IOError("The temporary directory has disappeared!") + return self._temp_dir + + def temp_file(self, prefix='file-', template='{prefix}{name}{suffix}', suffix='.tmp'): + """Generate the filename of a temporary file""" + filename = template.format(prefix=prefix, suffix=suffix, name=uuid.uuid4().hex) + return os.path.join(self.tempdir, filename) + + @classmethod + def data_file(cls, filename, *parts): + """Provide a data file from a filename, can accept directories as arguments.""" + if os.path.isabs(filename): + # Absolute root was passed in, so we trust that (it might be a tempdir) + full_path = os.path.join(filename, *parts) + else: + # Otherwise we assume it's relative to the test data dir. + full_path = os.path.join(cls.datadir(), filename, *parts) + + if not os.path.isfile(full_path): + raise IOError("Can't find test data file: {}".format(full_path)) + return full_path + + @property + def empty_svg(self): + """Returns a common minimal svg file""" + return self.data_file('svg', 'default-inkscape-SVG.svg') + + def assertAlmostTuple(self, found, expected, precision=8): # pylint: disable=invalid-name + """ + Floating point results may vary with computer architecture; use + assertAlmostEqual to allow a tolerance in the result. + """ + self.assertEqual(len(found), len(expected)) + for fon, exp in zip(found, expected): + self.assertAlmostEqual(fon, exp, precision) + + def assertEffectEmpty(self, effect, **kwargs): # pylint: disable=invalid-name + """Assert calling effect without any arguments""" + self.assertEffect(effect=effect, **kwargs) + + def assertEffect(self, *filename, **kwargs): # pylint: disable=invalid-name + """Assert an effect, capturing the output to stdout. + + filename should point to a starting svg document, default is empty_svg + """ + effect = kwargs.pop('effect', self.effect_class)() + + args = [self.data_file(*filename)] if filename else [self.empty_svg] # pylint: disable=no-value-for-parameter + args += kwargs.pop('args', []) + args += ['--{}={}'.format(*kw) for kw in kwargs.items()] + + # Output is redirected to this string io buffer + if self.stderr_output: + with Capture('stderr') as stderr: + effect.run(args, output=BytesIO()) + effect.test_output = stderr + else: + output = BytesIO() + with Capture('stdout', kwargs.get('stdout_protect', self.stdout_protect)) as stdout: + with Capture('stderr', kwargs.get('stderr_protect', self.stderr_protect)) as stderr: + effect.run(args, output=output) + self.assertEqual('', stdout.getvalue(), "Extra print statements detected") + self.assertEqual('', stderr.getvalue(), "Extra error or warnings detected") + effect.test_output = output + + if os.environ.get('FAIL_ON_DEPRECATION', False): + warnings = getattr(effect, 'warned_about', set()) + effect.warned_about = set() # reset for next test + self.assertFalse(warnings, "Deprecated API is still being used!") + + return effect + + def assertDeepAlmostEqual(self, first, second, places=None, msg=None, delta=None): + if delta is None and places is None: + places = 7 + if isinstance(first, (list, tuple)): + assert len(first) == len(second) + for (f, s) in zip(first, second): + self.assertDeepAlmostEqual(f, s, places, msg, delta) + else: + self.assertAlmostEqual(first, second, places, msg, delta) + + @property + def effect(self): + """Generate an effect object""" + if self._effect is None: + self._effect = self.effect_class() + return self._effect + +class InkscapeExtensionTestMixin(object): + """Automatically setup self.effect for each test and test with an empty svg""" + def setUp(self): # pylint: disable=invalid-name + """Check if there is an effect_class set and create self.effect if it is""" + super(InkscapeExtensionTestMixin, self).setUp() + if self.effect_class is None: + self.skipTest('self.effect_class is not defined for this this test') + + def test_default_settings(self): + """Extension works with empty svg file""" + self.effect.run([self.empty_svg]) + +class ComparisonMixin(object): + """ + Add comparison tests to any existing test suite. + """ + # This input svg file sent to the extension (if any) + compare_file = 'svg/shapes.svg' + # The ways in which the output is filtered for comparision (see filters.py) + compare_filters = [] # type: List[Compare] + # If true, the filtered output will be saved and only applied to the + # extension output (and not to the reference file) + compare_filter_save = False + # A list of comparison runs, each entry will cause the extension to be run. + comparisons = [ + (), + ('--id=p1', '--id=r3'), + ] + + def test_all_comparisons(self): + """Testing all comparisons""" + if not isinstance(self.compare_file, (list, tuple)): + self._test_comparisons(self.compare_file) + else: + for compare_file in self.compare_file: + self._test_comparisons( + compare_file, + addout=os.path.basename(compare_file) + ) + + def _test_comparisons(self, compare_file, addout=None): + for args in self.comparisons: + self.assertCompare( + compare_file, + self.get_compare_outfile(args, addout), + args, + ) + + def assertCompare(self, infile, outfile, args): #pylint: disable=invalid-name + """ + Compare the output of a previous run against this one. + + - infile: The filename of the pre-processed svg (or other type of file) + - outfile: The filename of the data we expect to get, if not set + the filename will be generated from the effect name and kwargs. + - args: All the arguments to be passed to the effect run + + """ + effect = self.assertEffect(infile, args=args) + + if outfile is None: + outfile = self.get_compare_outfile(args) + + if not os.path.isfile(outfile): + raise IOError("Comparison file {} not found".format(outfile)) + + data_a = effect.test_output.getvalue() + if os.environ.get('EXPORT_COMPARE', False): + with open(outfile + '.export', 'wb') as fhl: + if sys.version_info[0] == 3 and isinstance(data_a, str): + data_a = data_a.encode('utf-8') + fhl.write(self._apply_compare_filters(data_a, True)) + print("Written output: {}.export".format(outfile)) + + data_a = self._apply_compare_filters(data_a) + + with open(outfile, 'rb') as fhl: + data_b = self._apply_compare_filters(fhl.read(), False) + + if isinstance(data_a, bytes) and isinstance(data_b, bytes) \ + and data_a.startswith(b'<') and data_b.startswith(b'<'): + # Late importing + diff_xml, delta = xmldiff(data_a, data_b) + if not delta and not os.environ.get('EXPORT_COMPARE', False): + print('The XML is different, you can save the output using the EXPORT_COMPARE=1'\ + ' envionment variable. This will save the compared file as a ".output" file'\ + ' next to the reference file used in the test.\n') + diff = 'SVG Differences: {}\n\n'.format(outfile) + if os.environ.get('XML_DIFF', False): + diff = '<- ' + diff_xml + else: + for x, (value_a, value_b) in enumerate(delta): + try: + # Take advantage of better text diff in testcase's own asserts. + self.assertEqual(value_a, value_b) + except AssertionError as err: + diff += " {}. {}\n".format(x, str(err)) + self.assertTrue(delta, diff) + else: + # compare any content (non svg) + self.assertEqual(data_a, data_b) + + def _apply_compare_filters(self, data, is_saving=None): + data = to_bytes(data) + # Applying filters flips depending if we are saving the filtered content + # to disk, or filtering during the test run. This is because some filters + # are destructive others are useful for diagnostics. + if is_saving is self.compare_filter_save or is_saving is None: + for cfilter in self.compare_filters: + data = cfilter(data) + return data + + def get_compare_outfile(self, args, addout=None): + """Generate an output file for the arguments given""" + if addout is not None: + args = list(args) + [str(addout)] + opstr = '__'.join(args)\ + .replace(self.tempdir, 'TMP_DIR')\ + .replace(self.datadir(), 'DAT_DIR') + opstr = re.sub(r'[^\w-]', '__', opstr) + if opstr: + if len(opstr) > 127: + # avoid filename-too-long error + opstr = hashlib.md5(opstr.encode('latin1')).hexdigest() + opstr = '__' + opstr + return self.data_file("refs", "{}{}.out".format(self.effect_name, opstr)) |