diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 19:33:14 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 19:33:14 +0000 |
commit | 36d22d82aa202bb199967e9512281e9a53db42c9 (patch) | |
tree | 105e8c98ddea1c1e4784a60a5a6410fa416be2de /third_party/libwebrtc/build/skia_gold_common | |
parent | Initial commit. (diff) | |
download | firefox-esr-upstream.tar.xz firefox-esr-upstream.zip |
Adding upstream version 115.7.0esr.upstream/115.7.0esrupstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'third_party/libwebrtc/build/skia_gold_common')
14 files changed, 2315 insertions, 0 deletions
diff --git a/third_party/libwebrtc/build/skia_gold_common/.style.yapf b/third_party/libwebrtc/build/skia_gold_common/.style.yapf new file mode 100644 index 0000000000..239e0a247f --- /dev/null +++ b/third_party/libwebrtc/build/skia_gold_common/.style.yapf @@ -0,0 +1,6 @@ +[style] +based_on_style = pep8 + +column_limit = 80 +indent_width = 2 + diff --git a/third_party/libwebrtc/build/skia_gold_common/OWNERS b/third_party/libwebrtc/build/skia_gold_common/OWNERS new file mode 100644 index 0000000000..428f610282 --- /dev/null +++ b/third_party/libwebrtc/build/skia_gold_common/OWNERS @@ -0,0 +1 @@ +bsheedy@chromium.org diff --git a/third_party/libwebrtc/build/skia_gold_common/PRESUBMIT.py b/third_party/libwebrtc/build/skia_gold_common/PRESUBMIT.py new file mode 100644 index 0000000000..f4aeda79cf --- /dev/null +++ b/third_party/libwebrtc/build/skia_gold_common/PRESUBMIT.py @@ -0,0 +1,36 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Presubmit script for //build/skia_gold_common/. + +See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts +for more details on the presubmit API built into depot_tools. +""" + +USE_PYTHON3 = True + + +def CommonChecks(input_api, output_api): + output = [] + build_path = input_api.os_path.join(input_api.PresubmitLocalPath(), '..') + skia_gold_env = dict(input_api.environ) + skia_gold_env.update({ + 'PYTHONPATH': build_path, + 'PYTHONDONTWRITEBYTECODE': '1', + }) + output.extend( + input_api.canned_checks.RunUnitTestsInDirectory( + input_api, + output_api, + input_api.PresubmitLocalPath(), [r'^.+_unittest\.py$'], + env=skia_gold_env)) + output.extend(input_api.canned_checks.RunPylint(input_api, output_api)) + return output + + +def CheckChangeOnUpload(input_api, output_api): + return CommonChecks(input_api, output_api) + + +def CheckChangeOnCommit(input_api, output_api): + return CommonChecks(input_api, output_api) diff --git a/third_party/libwebrtc/build/skia_gold_common/README.md b/third_party/libwebrtc/build/skia_gold_common/README.md new file mode 100644 index 0000000000..ec72111748 --- /dev/null +++ b/third_party/libwebrtc/build/skia_gold_common/README.md @@ -0,0 +1,6 @@ +This directory contains Python code used for interacting with the Skia Gold +image diff service. It is used by multiple test harnesses, e.g. +`//build/android/test_runner.py` and +`//content/test/gpu/run_gpu_integration_test.py`. A place such as +`//testing/` would likely be a better location, but causes issues with +V8 since it imports `//build/` but not all of Chromium src. diff --git a/third_party/libwebrtc/build/skia_gold_common/__init__.py b/third_party/libwebrtc/build/skia_gold_common/__init__.py new file mode 100644 index 0000000000..ae1922e1cc --- /dev/null +++ b/third_party/libwebrtc/build/skia_gold_common/__init__.py @@ -0,0 +1,3 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. diff --git a/third_party/libwebrtc/build/skia_gold_common/output_managerless_skia_gold_session.py b/third_party/libwebrtc/build/skia_gold_common/output_managerless_skia_gold_session.py new file mode 100644 index 0000000000..b22487f749 --- /dev/null +++ b/third_party/libwebrtc/build/skia_gold_common/output_managerless_skia_gold_session.py @@ -0,0 +1,68 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Implementation of skia_gold_session.py without output managers. + +Diff output is instead stored in a directory and pointed to with file:// URLs. +""" + +import os +import subprocess +import time + +from skia_gold_common import skia_gold_session + + +class OutputManagerlessSkiaGoldSession(skia_gold_session.SkiaGoldSession): + def RunComparison( # pylint: disable=too-many-arguments + self, + name, + png_file, + output_manager=True, + inexact_matching_args=None, + use_luci=True, + optional_keys=None, + force_dryrun=False): + # Passing True for the output manager is a bit of a hack, as we don't + # actually need an output manager and just need to get past the truthy + # check. + return super(OutputManagerlessSkiaGoldSession, self).RunComparison( + name=name, + png_file=png_file, + output_manager=output_manager, + inexact_matching_args=inexact_matching_args, + use_luci=use_luci, + optional_keys=optional_keys, + force_dryrun=force_dryrun) + + def _CreateDiffOutputDir(self, name): + # Do this instead of just making a temporary directory so that it's easier + # for users to look through multiple results. We intentionally do not clean + # this directory up since the user might need to look at it later. + timestamp = int(time.time()) + name = '%s_%d' % (name, timestamp) + filepath = os.path.join(self._local_png_directory, name) + os.makedirs(filepath) + return filepath + + def _StoreDiffLinks(self, image_name, _, output_dir): + results = self._comparison_results.setdefault(image_name, + self.ComparisonResults()) + # The directory should contain "input-<hash>.png", "closest-<hash>.png", + # and "diff.png". + for f in os.listdir(output_dir): + file_url = 'file://%s' % os.path.join(output_dir, f) + if f.startswith('input-'): + results.local_diff_given_image = file_url + elif f.startswith('closest-'): + results.local_diff_closest_image = file_url + elif f == 'diff.png': + results.local_diff_diff_image = file_url + + @staticmethod + def _RunCmdForRcAndOutput(cmd): + try: + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + return 0, output + except subprocess.CalledProcessError as e: + return e.returncode, e.output diff --git a/third_party/libwebrtc/build/skia_gold_common/output_managerless_skia_gold_session_unittest.py b/third_party/libwebrtc/build/skia_gold_common/output_managerless_skia_gold_session_unittest.py new file mode 100755 index 0000000000..80937bb8e8 --- /dev/null +++ b/third_party/libwebrtc/build/skia_gold_common/output_managerless_skia_gold_session_unittest.py @@ -0,0 +1,137 @@ +#!/usr/bin/env vpython3 +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +#pylint: disable=protected-access + +import os +import re +import sys +import tempfile +import unittest + +import six + +if six.PY2: + import mock +else: + import unittest.mock as mock + +from pyfakefs import fake_filesystem_unittest + +from skia_gold_common import output_managerless_skia_gold_session as omsgs +from skia_gold_common import skia_gold_properties +from skia_gold_common import unittest_utils + +createSkiaGoldArgs = unittest_utils.createSkiaGoldArgs + + +def assertArgWith(test, arg_list, arg, value): + i = arg_list.index(arg) + test.assertEqual(arg_list[i + 1], value) + + +class GpuSkiaGoldSessionDiffTest(fake_filesystem_unittest.TestCase): + def setUp(self): + self.setUpPyfakefs() + self._working_dir = tempfile.mkdtemp() + self._json_keys = tempfile.NamedTemporaryFile(delete=False).name + + @mock.patch.object(omsgs.OutputManagerlessSkiaGoldSession, + '_RunCmdForRcAndOutput') + def test_commandCommonArgs(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=False) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = omsgs.OutputManagerlessSkiaGoldSession(self._working_dir, + sgp, + self._json_keys, + 'corpus', + instance='instance') + session.Diff('name', 'png_file', None) + call_args = cmd_mock.call_args[0][0] + self.assertIn('diff', call_args) + assertArgWith(self, call_args, '--corpus', 'corpus') + # TODO(skbug.com/10610): Remove the -public once we go back to using the + # non-public instance, or add a second test for testing that the correct + # instance is chosen if we decide to support both depending on what the + # user is authenticated for. + assertArgWith(self, call_args, '--instance', 'instance-public') + assertArgWith(self, call_args, '--input', 'png_file') + assertArgWith(self, call_args, '--test', 'name') + # TODO(skbug.com/10611): Re-add this assert and remove the check for the + # absence of the directory once we switch back to using the proper working + # directory. + # assertArgWith(self, call_args, '--work-dir', self._working_dir) + self.assertNotIn(self._working_dir, call_args) + i = call_args.index('--out-dir') + # The output directory should not be a subdirectory of the working + # directory. + self.assertNotIn(self._working_dir, call_args[i + 1]) + + @mock.patch.object(omsgs.OutputManagerlessSkiaGoldSession, '_StoreDiffLinks') + @mock.patch.object(omsgs.OutputManagerlessSkiaGoldSession, + '_RunCmdForRcAndOutput') + def test_explicitLocalPngDirectory(self, cmd_mock, _): + cmd_mock.return_value = (0, '') + if sys.platform == 'win32': + local_png_dir = 'c:\\tmp\\foo' + else: + local_png_dir = '/tmp/foo' + args = createSkiaGoldArgs(git_revision='a', + skia_gold_local_png_write_directory=local_png_dir) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = omsgs.OutputManagerlessSkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, + None) + _, _ = session.Diff('name', None, None) + self.assertEqual(cmd_mock.call_count, 1) + if six.PY3: + call_args = cmd_mock.call_args.args[0] + else: + call_args = cmd_mock.call_args[0][0] + self.assertIn('--out-dir', call_args) + output_dir = call_args[call_args.index('--out-dir') + 1] + # Directory should be a subdirectory of the directory we gave and be made + # up of the image name and a timestamp. + parent_dir, sub_dir = output_dir.rsplit(os.sep, 1) + self.assertEqual(parent_dir, local_png_dir) + sub_dir = os.path.normpath(sub_dir) + self.assertIsNotNone(re.match(r'^name_\d+$', sub_dir)) + + +class OutputManagerlessSkiaGoldSessionStoreDiffLinksTest( + fake_filesystem_unittest.TestCase): + def setUp(self): + self.setUpPyfakefs() + self._working_dir = tempfile.mkdtemp() + self._json_keys = tempfile.NamedTemporaryFile(delete=False).name + + def test_outputManagerNotNeeded(self): + args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=True) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = omsgs.OutputManagerlessSkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, + None) + input_filepath = os.path.join(self._working_dir, 'input-inputhash.png') + with open(input_filepath, 'w') as f: + f.write('') + closest_filepath = os.path.join(self._working_dir, + 'closest-closesthash.png') + with open(closest_filepath, 'w') as f: + f.write('') + diff_filepath = os.path.join(self._working_dir, 'diff.png') + with open(diff_filepath, 'w') as f: + f.write('') + + session._StoreDiffLinks('foo', None, self._working_dir) + self.assertEqual(session.GetGivenImageLink('foo'), + 'file://' + input_filepath) + self.assertEqual(session.GetClosestImageLink('foo'), + 'file://' + closest_filepath) + self.assertEqual(session.GetDiffImageLink('foo'), 'file://' + diff_filepath) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/third_party/libwebrtc/build/skia_gold_common/skia_gold_properties.py b/third_party/libwebrtc/build/skia_gold_common/skia_gold_properties.py new file mode 100644 index 0000000000..c34146c699 --- /dev/null +++ b/third_party/libwebrtc/build/skia_gold_common/skia_gold_properties.py @@ -0,0 +1,148 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Class for storing Skia Gold comparison properties. + +Examples: +* git revision being tested +* Whether the test is being run locally or on a bot +* What the continuous integration system is +""" + +import logging +import os + + +class SkiaGoldProperties(object): + def __init__(self, args): + """Abstract class to validate and store properties related to Skia Gold. + + Args: + args: The parsed arguments from an argparse.ArgumentParser. + """ + self._git_revision = None + self._issue = None + self._patchset = None + self._job_id = None + self._local_pixel_tests = None + self._no_luci_auth = None + self._bypass_skia_gold_functionality = None + self._code_review_system = None + self._continuous_integration_system = None + self._local_png_directory = None + + self._InitializeProperties(args) + + def IsTryjobRun(self): + return self.issue is not None + + @property + def continuous_integration_system(self): + return self._continuous_integration_system or 'buildbucket' + + @property + def code_review_system(self): + return self._code_review_system or 'gerrit' + + @property + def git_revision(self): + return self._GetGitRevision() + + @property + def issue(self): + return self._issue + + @property + def job_id(self): + return self._job_id + + @property + def local_pixel_tests(self): + return self._IsLocalRun() + + @property + def local_png_directory(self): + return self._local_png_directory + + @property + def no_luci_auth(self): + return self._no_luci_auth + + @property + def patchset(self): + return self._patchset + + @property + def bypass_skia_gold_functionality(self): + return self._bypass_skia_gold_functionality + + @staticmethod + def _GetGitOriginMasterHeadSha1(): + raise NotImplementedError() + + def _GetGitRevision(self): + if not self._git_revision: + # Automated tests should always pass the revision, so assume we're on + # a workstation and try to get the local origin/master HEAD. + if not self._IsLocalRun(): + raise RuntimeError( + '--git-revision was not passed when running on a bot') + revision = self._GetGitOriginMasterHeadSha1() + if not revision or len(revision) != 40: + raise RuntimeError( + '--git-revision not passed and unable to determine from git') + self._git_revision = revision + return self._git_revision + + def _IsLocalRun(self): + if self._local_pixel_tests is None: + # Look for the presence of the SWARMING_SERVER environment variable as a + # heuristic to determine whether we're running on a workstation or a bot. + # This should always be set on swarming, but would be strange to be set on + # a workstation. + self._local_pixel_tests = 'SWARMING_SERVER' not in os.environ + if self._local_pixel_tests: + logging.warning( + 'Automatically determined that test is running on a workstation') + else: + logging.warning( + 'Automatically determined that test is running on a bot') + return self._local_pixel_tests + + def _InitializeProperties(self, args): + if hasattr(args, 'local_pixel_tests'): + # If not set, will be automatically determined later if needed. + self._local_pixel_tests = args.local_pixel_tests + + if hasattr(args, 'skia_gold_local_png_write_directory'): + self._local_png_directory = args.skia_gold_local_png_write_directory + + if hasattr(args, 'no_luci_auth'): + self._no_luci_auth = args.no_luci_auth + + if hasattr(args, 'bypass_skia_gold_functionality'): + self._bypass_skia_gold_functionality = args.bypass_skia_gold_functionality + + if hasattr(args, 'code_review_system'): + self._code_review_system = args.code_review_system + + if hasattr(args, 'continuous_integration_system'): + self._continuous_integration_system = args.continuous_integration_system + + # Will be automatically determined later if needed. + if not hasattr(args, 'git_revision') or not args.git_revision: + return + self._git_revision = args.git_revision + + # Only expected on tryjob runs. + if not hasattr(args, 'gerrit_issue') or not args.gerrit_issue: + return + self._issue = args.gerrit_issue + if not hasattr(args, 'gerrit_patchset') or not args.gerrit_patchset: + raise RuntimeError( + '--gerrit-issue passed, but --gerrit-patchset not passed.') + self._patchset = args.gerrit_patchset + if not hasattr(args, 'buildbucket_id') or not args.buildbucket_id: + raise RuntimeError( + '--gerrit-issue passed, but --buildbucket-id not passed.') + self._job_id = args.buildbucket_id diff --git a/third_party/libwebrtc/build/skia_gold_common/skia_gold_properties_unittest.py b/third_party/libwebrtc/build/skia_gold_common/skia_gold_properties_unittest.py new file mode 100755 index 0000000000..ddbac3277d --- /dev/null +++ b/third_party/libwebrtc/build/skia_gold_common/skia_gold_properties_unittest.py @@ -0,0 +1,187 @@ +#!/usr/bin/env vpython3 +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +#pylint: disable=protected-access + +import os +import sys +import unittest + +if sys.version_info[0] == 2: + import mock +else: + import unittest.mock as mock + +from skia_gold_common import skia_gold_properties +from skia_gold_common import unittest_utils + +createSkiaGoldArgs = unittest_utils.createSkiaGoldArgs + + +class SkiaGoldPropertiesInitializationTest(unittest.TestCase): + """Tests that SkiaGoldProperties initializes (or doesn't) when expected.""" + + def verifySkiaGoldProperties(self, instance, expected): + self.assertEqual(instance._local_pixel_tests, + expected.get('local_pixel_tests')) + self.assertEqual(instance._no_luci_auth, expected.get('no_luci_auth')) + self.assertEqual(instance._code_review_system, + expected.get('code_review_system')) + self.assertEqual(instance._continuous_integration_system, + expected.get('continuous_integration_system')) + self.assertEqual(instance._git_revision, expected.get('git_revision')) + self.assertEqual(instance._issue, expected.get('gerrit_issue')) + self.assertEqual(instance._patchset, expected.get('gerrit_patchset')) + self.assertEqual(instance._job_id, expected.get('buildbucket_id')) + self.assertEqual(instance._bypass_skia_gold_functionality, + expected.get('bypass_skia_gold_functionality')) + + def test_initializeSkiaGoldAttributes_unsetLocal(self): + args = createSkiaGoldArgs() + sgp = skia_gold_properties.SkiaGoldProperties(args) + self.verifySkiaGoldProperties(sgp, {}) + + def test_initializeSkiaGoldAttributes_explicitLocal(self): + args = createSkiaGoldArgs(local_pixel_tests=True) + sgp = skia_gold_properties.SkiaGoldProperties(args) + self.verifySkiaGoldProperties(sgp, {'local_pixel_tests': True}) + + def test_initializeSkiaGoldAttributes_explicitNonLocal(self): + args = createSkiaGoldArgs(local_pixel_tests=False) + sgp = skia_gold_properties.SkiaGoldProperties(args) + self.verifySkiaGoldProperties(sgp, {'local_pixel_tests': False}) + + def test_initializeSkiaGoldAttributes_explicitNoLuciAuth(self): + args = createSkiaGoldArgs(no_luci_auth=True) + sgp = skia_gold_properties.SkiaGoldProperties(args) + self.verifySkiaGoldProperties(sgp, {'no_luci_auth': True}) + + def test_initializeSkiaGoldAttributes_explicitCrs(self): + args = createSkiaGoldArgs(code_review_system='foo') + sgp = skia_gold_properties.SkiaGoldProperties(args) + self.verifySkiaGoldProperties(sgp, {'code_review_system': 'foo'}) + + def test_initializeSkiaGoldAttributes_explicitCis(self): + args = createSkiaGoldArgs(continuous_integration_system='foo') + sgp = skia_gold_properties.SkiaGoldProperties(args) + self.verifySkiaGoldProperties(sgp, {'continuous_integration_system': 'foo'}) + + def test_initializeSkiaGoldAttributes_bypassExplicitTrue(self): + args = createSkiaGoldArgs(bypass_skia_gold_functionality=True) + sgp = skia_gold_properties.SkiaGoldProperties(args) + self.verifySkiaGoldProperties(sgp, {'bypass_skia_gold_functionality': True}) + + def test_initializeSkiaGoldAttributes_explicitGitRevision(self): + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + self.verifySkiaGoldProperties(sgp, {'git_revision': 'a'}) + + def test_initializeSkiaGoldAttributes_tryjobArgsIgnoredWithoutRevision(self): + args = createSkiaGoldArgs(gerrit_issue=1, + gerrit_patchset=2, + buildbucket_id=3) + sgp = skia_gold_properties.SkiaGoldProperties(args) + self.verifySkiaGoldProperties(sgp, {}) + + def test_initializeSkiaGoldAttributes_tryjobArgs(self): + args = createSkiaGoldArgs(git_revision='a', + gerrit_issue=1, + gerrit_patchset=2, + buildbucket_id=3) + sgp = skia_gold_properties.SkiaGoldProperties(args) + self.verifySkiaGoldProperties( + sgp, { + 'git_revision': 'a', + 'gerrit_issue': 1, + 'gerrit_patchset': 2, + 'buildbucket_id': 3 + }) + + def test_initializeSkiaGoldAttributes_tryjobMissingPatchset(self): + args = createSkiaGoldArgs(git_revision='a', + gerrit_issue=1, + buildbucket_id=3) + with self.assertRaises(RuntimeError): + skia_gold_properties.SkiaGoldProperties(args) + + def test_initializeSkiaGoldAttributes_tryjobMissingBuildbucket(self): + args = createSkiaGoldArgs(git_revision='a', + gerrit_issue=1, + gerrit_patchset=2) + with self.assertRaises(RuntimeError): + skia_gold_properties.SkiaGoldProperties(args) + + +class SkiaGoldPropertiesCalculationTest(unittest.TestCase): + """Tests that SkiaGoldProperties properly calculates certain properties.""" + + def testLocalPixelTests_determineTrue(self): + args = createSkiaGoldArgs() + sgp = skia_gold_properties.SkiaGoldProperties(args) + with mock.patch.dict(os.environ, {}, clear=True): + self.assertTrue(sgp.local_pixel_tests) + + def testLocalPixelTests_determineFalse(self): + args = createSkiaGoldArgs() + sgp = skia_gold_properties.SkiaGoldProperties(args) + with mock.patch.dict(os.environ, {'SWARMING_SERVER': ''}, clear=True): + self.assertFalse(sgp.local_pixel_tests) + + def testIsTryjobRun_noIssue(self): + args = createSkiaGoldArgs() + sgp = skia_gold_properties.SkiaGoldProperties(args) + self.assertFalse(sgp.IsTryjobRun()) + + def testIsTryjobRun_issue(self): + args = createSkiaGoldArgs(git_revision='a', + gerrit_issue=1, + gerrit_patchset=2, + buildbucket_id=3) + sgp = skia_gold_properties.SkiaGoldProperties(args) + self.assertTrue(sgp.IsTryjobRun()) + + def testGetGitRevision_revisionSet(self): + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + self.assertEqual(sgp.git_revision, 'a') + + def testGetGitRevision_findValidRevision(self): + args = createSkiaGoldArgs(local_pixel_tests=True) + sgp = skia_gold_properties.SkiaGoldProperties(args) + with mock.patch.object(skia_gold_properties.SkiaGoldProperties, + '_GetGitOriginMasterHeadSha1') as patched_head: + expected = 'a' * 40 + patched_head.return_value = expected + self.assertEqual(sgp.git_revision, expected) + # Should be cached. + self.assertEqual(sgp._git_revision, expected) + + def testGetGitRevision_noExplicitOnBot(self): + args = createSkiaGoldArgs(local_pixel_tests=False) + sgp = skia_gold_properties.SkiaGoldProperties(args) + with self.assertRaises(RuntimeError): + _ = sgp.git_revision + + def testGetGitRevision_findEmptyRevision(self): + args = createSkiaGoldArgs(local_pixel_tests=True) + sgp = skia_gold_properties.SkiaGoldProperties(args) + with mock.patch.object(skia_gold_properties.SkiaGoldProperties, + '_GetGitOriginMasterHeadSha1') as patched_head: + patched_head.return_value = '' + with self.assertRaises(RuntimeError): + _ = sgp.git_revision + + def testGetGitRevision_findMalformedRevision(self): + args = createSkiaGoldArgs(local_pixel_tests=True) + sgp = skia_gold_properties.SkiaGoldProperties(args) + with mock.patch.object(skia_gold_properties.SkiaGoldProperties, + '_GetGitOriginMasterHeadSha1') as patched_head: + patched_head.return_value = 'a' * 39 + with self.assertRaises(RuntimeError): + _ = sgp.git_revision + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/third_party/libwebrtc/build/skia_gold_common/skia_gold_session.py b/third_party/libwebrtc/build/skia_gold_common/skia_gold_session.py new file mode 100644 index 0000000000..7e69a238cd --- /dev/null +++ b/third_party/libwebrtc/build/skia_gold_common/skia_gold_session.py @@ -0,0 +1,552 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Class for interacting with the Skia Gold image diffing service.""" + +import logging +import os +import shutil +import sys +import tempfile +import time + +CHROMIUM_SRC = os.path.realpath( + os.path.join(os.path.dirname(__file__), '..', '..')) + +GOLDCTL_BINARY = os.path.join(CHROMIUM_SRC, 'tools', 'skia_goldctl') +if sys.platform == 'win32': + GOLDCTL_BINARY = os.path.join(GOLDCTL_BINARY, 'win', 'goldctl') + '.exe' +elif sys.platform == 'darwin': + GOLDCTL_BINARY = os.path.join(GOLDCTL_BINARY, 'mac', 'goldctl') +else: + GOLDCTL_BINARY = os.path.join(GOLDCTL_BINARY, 'linux', 'goldctl') + + +class SkiaGoldSession(object): + class StatusCodes(object): + """Status codes for RunComparison.""" + SUCCESS = 0 + AUTH_FAILURE = 1 + INIT_FAILURE = 2 + COMPARISON_FAILURE_REMOTE = 3 + COMPARISON_FAILURE_LOCAL = 4 + LOCAL_DIFF_FAILURE = 5 + NO_OUTPUT_MANAGER = 6 + + class ComparisonResults(object): + """Struct-like object for storing results of an image comparison.""" + + def __init__(self): + self.public_triage_link = None + self.internal_triage_link = None + self.triage_link_omission_reason = None + self.local_diff_given_image = None + self.local_diff_closest_image = None + self.local_diff_diff_image = None + + def __init__(self, + working_dir, + gold_properties, + keys_file, + corpus, + instance, + bucket=None): + """Abstract class to handle all aspects of image comparison via Skia Gold. + + A single SkiaGoldSession is valid for a single instance/corpus/keys_file + combination. + + Args: + working_dir: The directory to store config files, etc. + gold_properties: A skia_gold_properties.SkiaGoldProperties instance for + the current test run. + keys_file: A path to a JSON file containing various comparison config data + such as corpus and debug information like the hardware/software + configuration the images will be produced on. + corpus: The corpus that images that will be compared belong to. + instance: The name of the Skia Gold instance to interact with. + bucket: Overrides the formulaic Google Storage bucket name generated by + goldctl + """ + self._working_dir = working_dir + self._gold_properties = gold_properties + self._corpus = corpus + self._instance = instance + self._bucket = bucket + self._local_png_directory = (self._gold_properties.local_png_directory + or tempfile.mkdtemp()) + self._triage_link_file = tempfile.NamedTemporaryFile(suffix='.txt', + dir=working_dir, + delete=False).name + # A map of image name (string) to ComparisonResults for that image. + self._comparison_results = {} + self._authenticated = False + self._initialized = False + + # Copy the given keys file to the working directory in case it ends up + # getting deleted before we try to use it. + self._keys_file = os.path.join(working_dir, 'gold_keys.json') + shutil.copy(keys_file, self._keys_file) + + def RunComparison(self, + name, + png_file, + output_manager, + inexact_matching_args=None, + use_luci=True, + optional_keys=None, + force_dryrun=False): + """Helper method to run all steps to compare a produced image. + + Handles authentication, itnitialization, comparison, and, if necessary, + local diffing. + + Args: + name: The name of the image being compared. + png_file: A path to a PNG file containing the image to be compared. + output_manager: An output manager to use to store diff links. The + argument's type depends on what type a subclasses' _StoreDiffLinks + implementation expects. Can be None even if _StoreDiffLinks expects + a valid input, but will fail if it ever actually needs to be used. + inexact_matching_args: A list of strings containing extra command line + arguments to pass to Gold for inexact matching. Can be omitted to use + exact matching. + use_luci: If true, authentication will use the service account provided by + the LUCI context. If false, will attempt to use whatever is set up in + gsutil, which is only supported for local runs. + optional_keys: A dict containing optional key/value pairs to pass to Gold + for this comparison. Optional keys are keys unrelated to the + configuration the image was produced on, e.g. a comment or whether + Gold should treat the image as ignored. + force_dryrun: A boolean denoting whether dryrun should be forced on + regardless of whether this is a local comparison or not. + + Returns: + A tuple (status, error). |status| is a value from + SkiaGoldSession.StatusCodes signifying the result of the comparison. + |error| is an error message describing the status if not successful. + """ + auth_rc, auth_stdout = self.Authenticate(use_luci=use_luci) + if auth_rc: + return self.StatusCodes.AUTH_FAILURE, auth_stdout + + init_rc, init_stdout = self.Initialize() + if init_rc: + return self.StatusCodes.INIT_FAILURE, init_stdout + + compare_rc, compare_stdout = self.Compare( + name=name, + png_file=png_file, + inexact_matching_args=inexact_matching_args, + optional_keys=optional_keys, + force_dryrun=force_dryrun) + if not compare_rc: + return self.StatusCodes.SUCCESS, None + + logging.error('Gold comparison failed: %s', compare_stdout) + if not self._gold_properties.local_pixel_tests: + return self.StatusCodes.COMPARISON_FAILURE_REMOTE, compare_stdout + + if not output_manager: + return (self.StatusCodes.NO_OUTPUT_MANAGER, + 'No output manager for local diff images') + + diff_rc, diff_stdout = self.Diff(name=name, + png_file=png_file, + output_manager=output_manager) + if diff_rc: + return self.StatusCodes.LOCAL_DIFF_FAILURE, diff_stdout + return self.StatusCodes.COMPARISON_FAILURE_LOCAL, compare_stdout + + def Authenticate(self, use_luci=True): + """Authenticates with Skia Gold for this session. + + Args: + use_luci: If true, authentication will use the service account provided + by the LUCI context. If false, will attempt to use whatever is set up + in gsutil, which is only supported for local runs. + + Returns: + A tuple (return_code, output). |return_code| is the return code of the + authentication process. |output| is the stdout + stderr of the + authentication process. + """ + if self._authenticated: + return 0, None + if self._gold_properties.bypass_skia_gold_functionality: + logging.warning('Not actually authenticating with Gold due to ' + '--bypass-skia-gold-functionality being present.') + return 0, None + + auth_cmd = [GOLDCTL_BINARY, 'auth', '--work-dir', self._working_dir] + if use_luci: + auth_cmd.append('--luci') + elif not self._gold_properties.local_pixel_tests: + raise RuntimeError( + 'Cannot authenticate to Skia Gold with use_luci=False unless running ' + 'local pixel tests') + + rc, stdout = self._RunCmdForRcAndOutput(auth_cmd) + if rc == 0: + self._authenticated = True + return rc, stdout + + def Initialize(self): + """Initializes the working directory if necessary. + + This can technically be skipped if the same information is passed to the + command used for image comparison, but that is less efficient under the + hood. Doing it that way effectively requires an initialization for every + comparison (~250 ms) instead of once at the beginning. + + Returns: + A tuple (return_code, output). |return_code| is the return code of the + initialization process. |output| is the stdout + stderr of the + initialization process. + """ + if self._initialized: + return 0, None + if self._gold_properties.bypass_skia_gold_functionality: + logging.warning('Not actually initializing Gold due to ' + '--bypass-skia-gold-functionality being present.') + return 0, None + + init_cmd = [ + GOLDCTL_BINARY, + 'imgtest', + 'init', + '--passfail', + '--instance', + self._instance, + '--corpus', + self._corpus, + '--keys-file', + self._keys_file, + '--work-dir', + self._working_dir, + '--failure-file', + self._triage_link_file, + '--commit', + self._gold_properties.git_revision, + ] + if self._bucket: + init_cmd.extend(['--bucket', self._bucket]) + if self._gold_properties.IsTryjobRun(): + init_cmd.extend([ + '--issue', + str(self._gold_properties.issue), + '--patchset', + str(self._gold_properties.patchset), + '--jobid', + str(self._gold_properties.job_id), + '--crs', + str(self._gold_properties.code_review_system), + '--cis', + str(self._gold_properties.continuous_integration_system), + ]) + + rc, stdout = self._RunCmdForRcAndOutput(init_cmd) + if rc == 0: + self._initialized = True + return rc, stdout + + def Compare(self, + name, + png_file, + inexact_matching_args=None, + optional_keys=None, + force_dryrun=False): + """Compares the given image to images known to Gold. + + Triage links can later be retrieved using GetTriageLinks(). + + Args: + name: The name of the image being compared. + png_file: A path to a PNG file containing the image to be compared. + inexact_matching_args: A list of strings containing extra command line + arguments to pass to Gold for inexact matching. Can be omitted to use + exact matching. + optional_keys: A dict containing optional key/value pairs to pass to Gold + for this comparison. Optional keys are keys unrelated to the + configuration the image was produced on, e.g. a comment or whether + Gold should treat the image as ignored. + force_dryrun: A boolean denoting whether dryrun should be forced on + regardless of whether this is a local comparison or not. + + Returns: + A tuple (return_code, output). |return_code| is the return code of the + comparison process. |output| is the stdout + stderr of the comparison + process. + """ + if self._gold_properties.bypass_skia_gold_functionality: + logging.warning('Not actually comparing with Gold due to ' + '--bypass-skia-gold-functionality being present.') + return 0, None + + compare_cmd = [ + GOLDCTL_BINARY, + 'imgtest', + 'add', + '--test-name', + name, + '--png-file', + png_file, + '--work-dir', + self._working_dir, + ] + if self._gold_properties.local_pixel_tests or force_dryrun: + compare_cmd.append('--dryrun') + if inexact_matching_args: + logging.info('Using inexact matching arguments for image %s: %s', name, + inexact_matching_args) + compare_cmd.extend(inexact_matching_args) + + optional_keys = optional_keys or {} + for k, v in optional_keys.items(): + compare_cmd.extend([ + '--add-test-optional-key', + '%s:%s' % (k, v), + ]) + + self._ClearTriageLinkFile() + rc, stdout = self._RunCmdForRcAndOutput(compare_cmd) + + self._comparison_results[name] = self.ComparisonResults() + if rc == 0: + self._comparison_results[name].triage_link_omission_reason = ( + 'Comparison succeeded, no triage link') + elif self._gold_properties.IsTryjobRun(): + cl_triage_link = ('https://{instance}-gold.skia.org/cl/{crs}/{issue}') + cl_triage_link = cl_triage_link.format( + instance=self._instance, + crs=self._gold_properties.code_review_system, + issue=self._gold_properties.issue) + self._comparison_results[name].internal_triage_link = cl_triage_link + self._comparison_results[name].public_triage_link =\ + self._GeneratePublicTriageLink(cl_triage_link) + else: + try: + with open(self._triage_link_file) as tlf: + triage_link = tlf.read().strip() + if not triage_link: + self._comparison_results[name].triage_link_omission_reason = ( + 'Gold did not provide a triage link. This is likely a bug on ' + "Gold's end.") + self._comparison_results[name].internal_triage_link = None + self._comparison_results[name].public_triage_link = None + else: + self._comparison_results[name].internal_triage_link = triage_link + self._comparison_results[name].public_triage_link =\ + self._GeneratePublicTriageLink(triage_link) + except IOError: + self._comparison_results[name].triage_link_omission_reason = ( + 'Failed to read triage link from file') + return rc, stdout + + def Diff(self, name, png_file, output_manager): + """Performs a local image diff against the closest known positive in Gold. + + This is used for running tests on a workstation, where uploading data to + Gold for ingestion is not allowed, and thus the web UI is not available. + + Image links can later be retrieved using Get*ImageLink(). + + Args: + name: The name of the image being compared. + png_file: The path to a PNG file containing the image to be diffed. + output_manager: An output manager to use to store diff links. The + argument's type depends on what type a subclasses' _StoreDiffLinks + implementation expects. + + Returns: + A tuple (return_code, output). |return_code| is the return code of the + diff process. |output| is the stdout + stderr of the diff process. + """ + # Instead of returning that everything is okay and putting in dummy links, + # just fail since this should only be called when running locally and + # --bypass-skia-gold-functionality is only meant for use on the bots. + if self._gold_properties.bypass_skia_gold_functionality: + raise RuntimeError( + '--bypass-skia-gold-functionality is not supported when running ' + 'tests locally.') + + output_dir = self._CreateDiffOutputDir(name) + # TODO(skbug.com/10611): Remove this temporary work dir and instead just use + # self._working_dir once `goldctl diff` stops clobbering the auth files in + # the provided work directory. + temp_work_dir = tempfile.mkdtemp() + # shutil.copytree() fails if the destination already exists, so use a + # subdirectory of the temporary directory. + temp_work_dir = os.path.join(temp_work_dir, 'diff_work_dir') + try: + shutil.copytree(self._working_dir, temp_work_dir) + diff_cmd = [ + GOLDCTL_BINARY, + 'diff', + '--corpus', + self._corpus, + '--instance', + self._GetDiffGoldInstance(), + '--input', + png_file, + '--test', + name, + '--work-dir', + temp_work_dir, + '--out-dir', + output_dir, + ] + rc, stdout = self._RunCmdForRcAndOutput(diff_cmd) + self._StoreDiffLinks(name, output_manager, output_dir) + return rc, stdout + finally: + shutil.rmtree(os.path.realpath(os.path.join(temp_work_dir, '..'))) + + def GetTriageLinks(self, name): + """Gets the triage links for the given image. + + Args: + name: The name of the image to retrieve the triage link for. + + Returns: + A tuple (public, internal). |public| is a string containing the triage + link for the public Gold instance if it is available, or None if it is not + available for some reason. |internal| is the same as |public|, but + containing a link to the internal Gold instance. The reason for links not + being available can be retrieved using GetTriageLinkOmissionReason. + """ + comparison_results = self._comparison_results.get(name, + self.ComparisonResults()) + return (comparison_results.public_triage_link, + comparison_results.internal_triage_link) + + def GetTriageLinkOmissionReason(self, name): + """Gets the reason why a triage link is not available for an image. + + Args: + name: The name of the image whose triage link does not exist. + + Returns: + A string containing the reason why a triage link is not available. + """ + if name not in self._comparison_results: + return 'No image comparison performed for %s' % name + results = self._comparison_results[name] + # This method should not be called if there is a valid triage link. + assert results.public_triage_link is None + assert results.internal_triage_link is None + if results.triage_link_omission_reason: + return results.triage_link_omission_reason + if results.local_diff_given_image: + return 'Gold only used to do a local image diff' + raise RuntimeError( + 'Somehow have a ComparisonResults instance for %s that should not ' + 'exist' % name) + + def GetGivenImageLink(self, name): + """Gets the link to the given image used for local diffing. + + Args: + name: The name of the image that was diffed. + + Returns: + A string containing the link to where the image is saved, or None if it + does not exist. + """ + assert name in self._comparison_results + return self._comparison_results[name].local_diff_given_image + + def GetClosestImageLink(self, name): + """Gets the link to the closest known image used for local diffing. + + Args: + name: The name of the image that was diffed. + + Returns: + A string containing the link to where the image is saved, or None if it + does not exist. + """ + assert name in self._comparison_results + return self._comparison_results[name].local_diff_closest_image + + def GetDiffImageLink(self, name): + """Gets the link to the diff between the given and closest images. + + Args: + name: The name of the image that was diffed. + + Returns: + A string containing the link to where the image is saved, or None if it + does not exist. + """ + assert name in self._comparison_results + return self._comparison_results[name].local_diff_diff_image + + def _GeneratePublicTriageLink(self, internal_link): + """Generates a public triage link given an internal one. + + Args: + internal_link: A string containing a triage link pointing to an internal + Gold instance. + + Returns: + A string containing a triage link pointing to the public mirror of the + link pointed to by |internal_link|. + """ + return internal_link.replace('%s-gold' % self._instance, + '%s-public-gold' % self._instance) + + def _ClearTriageLinkFile(self): + """Clears the contents of the triage link file. + + This should be done before every comparison since goldctl appends to the + file instead of overwriting its contents, which results in multiple triage + links getting concatenated together if there are multiple failures. + """ + open(self._triage_link_file, 'w').close() + + def _CreateDiffOutputDir(self, _): + # We don't use self._local_png_directory here since we want it to be + # automatically cleaned up with the working directory. Any subclasses that + # want to keep it around can override this method. + return tempfile.mkdtemp(dir=self._working_dir) + + def _GetDiffGoldInstance(self): + """Gets the Skia Gold instance to use for the Diff step. + + This can differ based on how a particular instance is set up, mainly + depending on whether it is set up for internal results or not. + """ + # TODO(skbug.com/10610): Decide whether to use the public or + # non-public instance once authentication is fixed for the non-public + # instance. + return str(self._instance) + '-public' + + def _StoreDiffLinks(self, image_name, output_manager, output_dir): + """Stores the local diff files as links. + + The ComparisonResults entry for |image_name| should have its *_image fields + filled after this unless corresponding images were not found on disk. + + Args: + image_name: A string containing the name of the image that was diffed. + output_manager: An output manager used used to surface links to users, + if necessary. The expected argument type depends on each subclasses' + implementation of this method. + output_dir: A string containing the path to the directory where diff + output image files where saved. + """ + raise NotImplementedError() + + @staticmethod + def _RunCmdForRcAndOutput(cmd): + """Runs |cmd| and returns its returncode and output. + + Args: + cmd: A list containing the command line to run. + + Returns: + A tuple (rc, output), where, |rc| is the returncode of the command and + |output| is the stdout + stderr of the command. + """ + raise NotImplementedError() diff --git a/third_party/libwebrtc/build/skia_gold_common/skia_gold_session_manager.py b/third_party/libwebrtc/build/skia_gold_common/skia_gold_session_manager.py new file mode 100644 index 0000000000..d4166e1dc2 --- /dev/null +++ b/third_party/libwebrtc/build/skia_gold_common/skia_gold_session_manager.py @@ -0,0 +1,121 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Class for managing multiple SkiaGoldSessions.""" + +import json +import tempfile + + +class SkiaGoldSessionManager(object): + def __init__(self, working_dir, gold_properties): + """Abstract class to manage one or more skia_gold_session.SkiaGoldSessions. + + A separate session is required for each instance/corpus/keys_file + combination, so this class will lazily create them as necessary. + + Args: + working_dir: The working directory under which each individual + SkiaGoldSessions' working directory will be created. + gold_properties: A SkiaGoldProperties instance that will be used to create + any SkiaGoldSessions. + """ + self._working_dir = working_dir + self._gold_properties = gold_properties + self._sessions = {} + + def GetSkiaGoldSession(self, + keys_input, + corpus=None, + instance=None, + bucket=None): + """Gets a SkiaGoldSession for the given arguments. + + Lazily creates one if necessary. + + Args: + keys_input: A way of retrieving various comparison config data such as + corpus and debug information like the hardware/software configuration + the image was produced on. Can be either a dict or a filepath to a + file containing JSON to read. + corpus: A string containing the corpus the session is for. If None, the + corpus will be determined using available information. + instance: The name of the Skia Gold instance to interact with. If None, + will use whatever default the subclass sets. + bucket: Overrides the formulaic Google Storage bucket name generated by + goldctl + """ + instance = instance or self._GetDefaultInstance() + keys_dict = _GetKeysAsDict(keys_input) + keys_string = json.dumps(keys_dict, sort_keys=True) + if corpus is None: + corpus = keys_dict.get('source_type', instance) + # Use the string representation of the keys JSON as a proxy for a hash since + # dicts themselves are not hashable. + session = self._sessions.setdefault(instance, + {}).setdefault(corpus, {}).setdefault( + keys_string, None) + if not session: + working_dir = tempfile.mkdtemp(dir=self._working_dir) + keys_file = _GetKeysAsJson(keys_input, working_dir) + session = self.GetSessionClass()(working_dir, self._gold_properties, + keys_file, corpus, instance, bucket) + self._sessions[instance][corpus][keys_string] = session + return session + + @staticmethod + def _GetDefaultInstance(): + """Gets the default Skia Gold instance. + + Returns: + A string containing the default instance. + """ + return 'chrome' + + @staticmethod + def GetSessionClass(): + """Gets the SkiaGoldSession class to use for session creation. + + Returns: + A reference to a SkiaGoldSession class. + """ + raise NotImplementedError + + +def _GetKeysAsDict(keys_input): + """Converts |keys_input| into a dictionary. + + Args: + keys_input: A dictionary or a string pointing to a JSON file. The contents + of either should be Skia Gold config data. + + Returns: + A dictionary containing the Skia Gold config data. + """ + if isinstance(keys_input, dict): + return keys_input + assert isinstance(keys_input, str) + with open(keys_input) as f: + return json.load(f) + + +def _GetKeysAsJson(keys_input, session_work_dir): + """Converts |keys_input| into a JSON file on disk. + + Args: + keys_input: A dictionary or a string pointing to a JSON file. The contents + of either should be Skia Gold config data. + + Returns: + A string containing a filepath to a JSON file with containing |keys_input|'s + data. + """ + if isinstance(keys_input, str): + return keys_input + assert isinstance(keys_input, dict) + keys_file = tempfile.NamedTemporaryFile(suffix='.json', + dir=session_work_dir, + delete=False).name + with open(keys_file, 'w') as f: + json.dump(keys_input, f) + return keys_file diff --git a/third_party/libwebrtc/build/skia_gold_common/skia_gold_session_manager_unittest.py b/third_party/libwebrtc/build/skia_gold_common/skia_gold_session_manager_unittest.py new file mode 100755 index 0000000000..286fdf2b97 --- /dev/null +++ b/third_party/libwebrtc/build/skia_gold_common/skia_gold_session_manager_unittest.py @@ -0,0 +1,180 @@ +#!/usr/bin/env vpython3 +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +#pylint: disable=protected-access + +import json +import os +import sys +import tempfile +import unittest + +if sys.version_info[0] == 2: + import mock +else: + import unittest.mock as mock + +from pyfakefs import fake_filesystem_unittest + +from skia_gold_common import skia_gold_properties +from skia_gold_common import skia_gold_session +from skia_gold_common import skia_gold_session_manager +from skia_gold_common import unittest_utils + +createSkiaGoldArgs = unittest_utils.createSkiaGoldArgs + + +class SkiaGoldSessionManagerGetSessionTest(fake_filesystem_unittest.TestCase): + """Tests the functionality of SkiaGoldSessionManager.GetSkiaGoldSession.""" + + def setUp(self): + self.setUpPyfakefs() + self._working_dir = tempfile.mkdtemp() + self._patcher = mock.patch.object( + skia_gold_session_manager.SkiaGoldSessionManager, 'GetSessionClass') + self._session_class_mock = self._patcher.start() + self._session_class_mock.return_value = skia_gold_session.SkiaGoldSession + self.addCleanup(self._patcher.stop) + + def test_ArgsForwardedToSession(self): + args = createSkiaGoldArgs() + sgp = skia_gold_properties.SkiaGoldProperties(args) + sgsm = skia_gold_session_manager.SkiaGoldSessionManager( + self._working_dir, sgp) + session = sgsm.GetSkiaGoldSession({}, 'corpus', 'instance') + self.assertTrue(session._keys_file.startswith(self._working_dir)) + self.assertEqual(session._corpus, 'corpus') + self.assertEqual(session._instance, 'instance') + # Make sure the session's working directory is a subdirectory of the + # manager's working directory. + self.assertEqual(os.path.dirname(session._working_dir), self._working_dir) + + def test_corpusFromJson(self): + args = createSkiaGoldArgs() + sgp = skia_gold_properties.SkiaGoldProperties(args) + sgsm = skia_gold_session_manager.SkiaGoldSessionManager( + self._working_dir, sgp) + session = sgsm.GetSkiaGoldSession({'source_type': 'foobar'}, None, + 'instance') + self.assertTrue(session._keys_file.startswith(self._working_dir)) + self.assertEqual(session._corpus, 'foobar') + self.assertEqual(session._instance, 'instance') + + def test_corpusDefaultsToInstance(self): + args = createSkiaGoldArgs() + sgp = skia_gold_properties.SkiaGoldProperties(args) + sgsm = skia_gold_session_manager.SkiaGoldSessionManager( + self._working_dir, sgp) + session = sgsm.GetSkiaGoldSession({}, None, 'instance') + self.assertTrue(session._keys_file.startswith(self._working_dir)) + self.assertEqual(session._corpus, 'instance') + self.assertEqual(session._instance, 'instance') + + @mock.patch.object(skia_gold_session_manager.SkiaGoldSessionManager, + '_GetDefaultInstance') + def test_getDefaultInstance(self, default_instance_mock): + default_instance_mock.return_value = 'default' + args = createSkiaGoldArgs() + sgp = skia_gold_properties.SkiaGoldProperties(args) + sgsm = skia_gold_session_manager.SkiaGoldSessionManager( + self._working_dir, sgp) + session = sgsm.GetSkiaGoldSession({}, None, None) + self.assertTrue(session._keys_file.startswith(self._working_dir)) + self.assertEqual(session._corpus, 'default') + self.assertEqual(session._instance, 'default') + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '__init__') + def test_matchingSessionReused(self, session_mock): + session_mock.return_value = None + args = createSkiaGoldArgs() + sgp = skia_gold_properties.SkiaGoldProperties(args) + sgsm = skia_gold_session_manager.SkiaGoldSessionManager( + self._working_dir, sgp) + session1 = sgsm.GetSkiaGoldSession({}, 'corpus', 'instance') + session2 = sgsm.GetSkiaGoldSession({}, 'corpus', 'instance') + self.assertEqual(session1, session2) + # For some reason, session_mock.assert_called_once() always passes, + # so check the call count directly. + self.assertEqual(session_mock.call_count, 1) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '__init__') + def test_separateSessionsFromKeys(self, session_mock): + session_mock.return_value = None + args = createSkiaGoldArgs() + sgp = skia_gold_properties.SkiaGoldProperties(args) + sgsm = skia_gold_session_manager.SkiaGoldSessionManager( + self._working_dir, sgp) + session1 = sgsm.GetSkiaGoldSession({}, 'corpus', 'instance') + session2 = sgsm.GetSkiaGoldSession({'something_different': 1}, 'corpus', + 'instance') + self.assertNotEqual(session1, session2) + self.assertEqual(session_mock.call_count, 2) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '__init__') + def test_separateSessionsFromCorpus(self, session_mock): + session_mock.return_value = None + args = createSkiaGoldArgs() + sgp = skia_gold_properties.SkiaGoldProperties(args) + sgsm = skia_gold_session_manager.SkiaGoldSessionManager( + self._working_dir, sgp) + session1 = sgsm.GetSkiaGoldSession({}, 'corpus1', 'instance') + session2 = sgsm.GetSkiaGoldSession({}, 'corpus2', 'instance') + self.assertNotEqual(session1, session2) + self.assertEqual(session_mock.call_count, 2) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '__init__') + def test_separateSessionsFromInstance(self, session_mock): + session_mock.return_value = None + args = createSkiaGoldArgs() + sgp = skia_gold_properties.SkiaGoldProperties(args) + self._working_dir = tempfile.mkdtemp() + sgsm = skia_gold_session_manager.SkiaGoldSessionManager( + self._working_dir, sgp) + session1 = sgsm.GetSkiaGoldSession({}, 'corpus', 'instance1') + session2 = sgsm.GetSkiaGoldSession({}, 'corpus', 'instance2') + self.assertNotEqual(session1, session2) + self.assertEqual(session_mock.call_count, 2) + + +class SkiaGoldSessionManagerKeyConversionTest(fake_filesystem_unittest.TestCase + ): + def setUp(self): + self.setUpPyfakefs() + self._working_dir = tempfile.mkdtemp() + + def test_getKeysAsDict(self): + keys_dict = {'foo': 'bar'} + keys_file_contents = {'bar': 'baz'} + keys_file = tempfile.NamedTemporaryFile(delete=False).name + with open(keys_file, 'w') as f: + json.dump(keys_file_contents, f) + + self.assertEqual(skia_gold_session_manager._GetKeysAsDict(keys_dict), + keys_dict) + self.assertEqual(skia_gold_session_manager._GetKeysAsDict(keys_file), + keys_file_contents) + with self.assertRaises(AssertionError): + skia_gold_session_manager._GetKeysAsDict(1) + + def test_getKeysAsJson(self): + keys_dict = {'foo': 'bar'} + keys_file_contents = {'bar': 'baz'} + keys_file = tempfile.NamedTemporaryFile(delete=False).name + with open(keys_file, 'w') as f: + json.dump(keys_file_contents, f) + + self.assertEqual(skia_gold_session_manager._GetKeysAsJson(keys_file, None), + keys_file) + keys_dict_as_json = skia_gold_session_manager._GetKeysAsJson( + keys_dict, self._working_dir) + self.assertTrue(keys_dict_as_json.startswith(self._working_dir)) + with open(keys_dict_as_json) as f: + self.assertEqual(json.load(f), keys_dict) + with self.assertRaises(AssertionError): + skia_gold_session_manager._GetKeysAsJson(1, None) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/third_party/libwebrtc/build/skia_gold_common/skia_gold_session_unittest.py b/third_party/libwebrtc/build/skia_gold_common/skia_gold_session_unittest.py new file mode 100755 index 0000000000..15b8a9924c --- /dev/null +++ b/third_party/libwebrtc/build/skia_gold_common/skia_gold_session_unittest.py @@ -0,0 +1,834 @@ +#!/usr/bin/env vpython3 +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +#pylint: disable=protected-access + +import json +import os +import sys +import tempfile +import unittest + +if sys.version_info[0] == 2: + import mock +else: + import unittest.mock as mock + +from pyfakefs import fake_filesystem_unittest + +from skia_gold_common import skia_gold_properties +from skia_gold_common import skia_gold_session +from skia_gold_common import unittest_utils + +createSkiaGoldArgs = unittest_utils.createSkiaGoldArgs + + +def assertArgWith(test, arg_list, arg, value): + i = arg_list.index(arg) + test.assertEqual(arg_list[i + 1], value) + + +class SkiaGoldSessionRunComparisonTest(fake_filesystem_unittest.TestCase): + """Tests the functionality of SkiaGoldSession.RunComparison.""" + + def setUp(self): + self.setUpPyfakefs() + self._working_dir = tempfile.mkdtemp() + self._json_keys = tempfile.NamedTemporaryFile(delete=False).name + with open(self._json_keys, 'w') as f: + json.dump({}, f) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Diff') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Compare') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Initialize') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Authenticate') + def test_comparisonSuccess(self, auth_mock, init_mock, compare_mock, + diff_mock): + auth_mock.return_value = (0, None) + init_mock.return_value = (0, None) + compare_mock.return_value = (0, None) + sgp = skia_gold_properties.SkiaGoldProperties(createSkiaGoldArgs()) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + status, _ = session.RunComparison(None, None, None) + self.assertEqual(status, + skia_gold_session.SkiaGoldSession.StatusCodes.SUCCESS) + self.assertEqual(auth_mock.call_count, 1) + self.assertEqual(init_mock.call_count, 1) + self.assertEqual(compare_mock.call_count, 1) + self.assertEqual(diff_mock.call_count, 0) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Diff') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Compare') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Initialize') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Authenticate') + def test_authFailure(self, auth_mock, init_mock, compare_mock, diff_mock): + auth_mock.return_value = (1, 'Auth failed') + sgp = skia_gold_properties.SkiaGoldProperties(createSkiaGoldArgs()) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + status, error = session.RunComparison(None, None, None) + self.assertEqual(status, + skia_gold_session.SkiaGoldSession.StatusCodes.AUTH_FAILURE) + self.assertEqual(error, 'Auth failed') + self.assertEqual(auth_mock.call_count, 1) + self.assertEqual(init_mock.call_count, 0) + self.assertEqual(compare_mock.call_count, 0) + self.assertEqual(diff_mock.call_count, 0) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Diff') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Compare') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Initialize') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Authenticate') + def test_initFailure(self, auth_mock, init_mock, compare_mock, diff_mock): + auth_mock.return_value = (0, None) + init_mock.return_value = (1, 'Init failed') + sgp = skia_gold_properties.SkiaGoldProperties(createSkiaGoldArgs()) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + status, error = session.RunComparison(None, None, None) + self.assertEqual(status, + skia_gold_session.SkiaGoldSession.StatusCodes.INIT_FAILURE) + self.assertEqual(error, 'Init failed') + self.assertEqual(auth_mock.call_count, 1) + self.assertEqual(init_mock.call_count, 1) + self.assertEqual(compare_mock.call_count, 0) + self.assertEqual(diff_mock.call_count, 0) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Diff') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Compare') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Initialize') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Authenticate') + def test_compareFailureRemote(self, auth_mock, init_mock, compare_mock, + diff_mock): + auth_mock.return_value = (0, None) + init_mock.return_value = (0, None) + compare_mock.return_value = (1, 'Compare failed') + args = createSkiaGoldArgs(local_pixel_tests=False) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + status, error = session.RunComparison(None, None, None) + self.assertEqual( + status, + skia_gold_session.SkiaGoldSession.StatusCodes.COMPARISON_FAILURE_REMOTE) + self.assertEqual(error, 'Compare failed') + self.assertEqual(auth_mock.call_count, 1) + self.assertEqual(init_mock.call_count, 1) + self.assertEqual(compare_mock.call_count, 1) + self.assertEqual(diff_mock.call_count, 0) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Diff') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Compare') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Initialize') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Authenticate') + def test_compareFailureLocal(self, auth_mock, init_mock, compare_mock, + diff_mock): + auth_mock.return_value = (0, None) + init_mock.return_value = (0, None) + compare_mock.return_value = (1, 'Compare failed') + diff_mock.return_value = (0, None) + args = createSkiaGoldArgs(local_pixel_tests=True) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + status, error = session.RunComparison(None, None, + 'Definitely an output manager') + self.assertEqual( + status, + skia_gold_session.SkiaGoldSession.StatusCodes.COMPARISON_FAILURE_LOCAL) + self.assertEqual(error, 'Compare failed') + self.assertEqual(auth_mock.call_count, 1) + self.assertEqual(init_mock.call_count, 1) + self.assertEqual(compare_mock.call_count, 1) + self.assertEqual(diff_mock.call_count, 1) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Diff') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Compare') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Initialize') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Authenticate') + def test_compareInexactMatching(self, auth_mock, init_mock, compare_mock, + diff_mock): + auth_mock.return_value = (0, None) + init_mock.return_value = (0, None) + compare_mock.return_value = (0, None) + diff_mock.return_value = (0, None) + args = createSkiaGoldArgs(local_pixel_tests=False) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + status, _ = session.RunComparison(None, + None, + None, + inexact_matching_args=['--inexact']) + self.assertEqual(status, + skia_gold_session.SkiaGoldSession.StatusCodes.SUCCESS) + self.assertEqual(auth_mock.call_count, 1) + self.assertEqual(init_mock.call_count, 1) + self.assertEqual(compare_mock.call_count, 1) + self.assertEqual(diff_mock.call_count, 0) + compare_mock.assert_called_with(name=None, + png_file=mock.ANY, + inexact_matching_args=['--inexact'], + optional_keys=None, + force_dryrun=False) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Diff') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Compare') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Initialize') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Authenticate') + def test_compareOptionalKeys(self, auth_mock, init_mock, compare_mock, + diff_mock): + auth_mock.return_value = (0, None) + init_mock.return_value = (0, None) + compare_mock.return_value = (0, None) + diff_mock.return_value = (0, None) + args = createSkiaGoldArgs(local_pixel_tests=False) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + status, _ = session.RunComparison(None, + None, + None, + optional_keys={'foo': 'bar'}) + self.assertEqual(status, + skia_gold_session.SkiaGoldSession.StatusCodes.SUCCESS) + self.assertEqual(auth_mock.call_count, 1) + self.assertEqual(init_mock.call_count, 1) + self.assertEqual(compare_mock.call_count, 1) + self.assertEqual(diff_mock.call_count, 0) + compare_mock.assert_called_with(name=None, + png_file=mock.ANY, + inexact_matching_args=None, + optional_keys={'foo': 'bar'}, + force_dryrun=False) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Diff') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Compare') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Initialize') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Authenticate') + def test_compareForceDryrun(self, auth_mock, init_mock, compare_mock, + diff_mock): + auth_mock.return_value = (0, None) + init_mock.return_value = (0, None) + compare_mock.return_value = (0, None) + diff_mock.return_value = (0, None) + args = createSkiaGoldArgs(local_pixel_tests=False) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + status, _ = session.RunComparison(None, None, None, force_dryrun=True) + self.assertEqual(status, + skia_gold_session.SkiaGoldSession.StatusCodes.SUCCESS) + self.assertEqual(auth_mock.call_count, 1) + self.assertEqual(init_mock.call_count, 1) + self.assertEqual(compare_mock.call_count, 1) + self.assertEqual(diff_mock.call_count, 0) + compare_mock.assert_called_with(name=None, + png_file=mock.ANY, + inexact_matching_args=None, + optional_keys=None, + force_dryrun=True) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Diff') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Compare') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Initialize') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Authenticate') + def test_diffFailure(self, auth_mock, init_mock, compare_mock, diff_mock): + auth_mock.return_value = (0, None) + init_mock.return_value = (0, None) + compare_mock.return_value = (1, 'Compare failed') + diff_mock.return_value = (1, 'Diff failed') + args = createSkiaGoldArgs(local_pixel_tests=True) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + status, error = session.RunComparison(None, None, + 'Definitely an output manager') + self.assertEqual( + status, + skia_gold_session.SkiaGoldSession.StatusCodes.LOCAL_DIFF_FAILURE) + self.assertEqual(error, 'Diff failed') + self.assertEqual(auth_mock.call_count, 1) + self.assertEqual(init_mock.call_count, 1) + self.assertEqual(compare_mock.call_count, 1) + self.assertEqual(diff_mock.call_count, 1) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Diff') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Compare') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Initialize') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Authenticate') + def test_noOutputManagerLocal(self, auth_mock, init_mock, compare_mock, + diff_mock): + auth_mock.return_value = (0, None) + init_mock.return_value = (0, None) + compare_mock.return_value = (1, 'Compare failed') + diff_mock.return_value = (0, None) + args = createSkiaGoldArgs(local_pixel_tests=True) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + status, error = session.RunComparison(None, None, None) + self.assertEqual( + status, skia_gold_session.SkiaGoldSession.StatusCodes.NO_OUTPUT_MANAGER) + self.assertEqual(error, 'No output manager for local diff images') + self.assertEqual(auth_mock.call_count, 1) + self.assertEqual(compare_mock.call_count, 1) + self.assertEqual(diff_mock.call_count, 0) + + +class SkiaGoldSessionAuthenticateTest(fake_filesystem_unittest.TestCase): + """Tests the functionality of SkiaGoldSession.Authenticate.""" + + def setUp(self): + self.setUpPyfakefs() + self._working_dir = tempfile.mkdtemp() + self._json_keys = tempfile.NamedTemporaryFile(delete=False).name + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_commandOutputReturned(self, cmd_mock): + cmd_mock.return_value = (1, 'Something bad :(') + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + rc, stdout = session.Authenticate() + self.assertEqual(cmd_mock.call_count, 1) + self.assertEqual(rc, 1) + self.assertEqual(stdout, 'Something bad :(') + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_bypassSkiaGoldFunctionality(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a', + bypass_skia_gold_functionality=True) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + rc, _ = session.Authenticate() + self.assertEqual(rc, 0) + cmd_mock.assert_not_called() + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_shortCircuitAlreadyAuthenticated(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + session._authenticated = True + rc, _ = session.Authenticate() + self.assertEqual(rc, 0) + cmd_mock.assert_not_called() + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_successSetsShortCircuit(self, cmd_mock): + cmd_mock.return_value = (0, None) + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + self.assertFalse(session._authenticated) + rc, _ = session.Authenticate() + self.assertEqual(rc, 0) + self.assertTrue(session._authenticated) + cmd_mock.assert_called_once() + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_failureDoesNotSetShortCircuit(self, cmd_mock): + cmd_mock.return_value = (1, None) + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + self.assertFalse(session._authenticated) + rc, _ = session.Authenticate() + self.assertEqual(rc, 1) + self.assertFalse(session._authenticated) + cmd_mock.assert_called_once() + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_commandWithUseLuciTrue(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + session.Authenticate(use_luci=True) + self.assertIn('--luci', cmd_mock.call_args[0][0]) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_commandWithUseLuciFalse(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=True) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + session.Authenticate(use_luci=False) + self.assertNotIn('--luci', cmd_mock.call_args[0][0]) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_commandWithUseLuciFalseNotLocal(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=False) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + with self.assertRaises(RuntimeError): + session.Authenticate(use_luci=False) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_commandCommonArgs(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + session.Authenticate() + call_args = cmd_mock.call_args[0][0] + self.assertIn('auth', call_args) + assertArgWith(self, call_args, '--work-dir', self._working_dir) + + +class SkiaGoldSessionInitializeTest(fake_filesystem_unittest.TestCase): + """Tests the functionality of SkiaGoldSession.Initialize.""" + + def setUp(self): + self.setUpPyfakefs() + self._working_dir = tempfile.mkdtemp() + self._json_keys = tempfile.NamedTemporaryFile(delete=False).name + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_bypassSkiaGoldFunctionality(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a', + bypass_skia_gold_functionality=True) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + rc, _ = session.Initialize() + self.assertEqual(rc, 0) + cmd_mock.assert_not_called() + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_shortCircuitAlreadyInitialized(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + session._initialized = True + rc, _ = session.Initialize() + self.assertEqual(rc, 0) + cmd_mock.assert_not_called() + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_successSetsShortCircuit(self, cmd_mock): + cmd_mock.return_value = (0, None) + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + self.assertFalse(session._initialized) + rc, _ = session.Initialize() + self.assertEqual(rc, 0) + self.assertTrue(session._initialized) + cmd_mock.assert_called_once() + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_failureDoesNotSetShortCircuit(self, cmd_mock): + cmd_mock.return_value = (1, None) + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + self.assertFalse(session._initialized) + rc, _ = session.Initialize() + self.assertEqual(rc, 1) + self.assertFalse(session._initialized) + cmd_mock.assert_called_once() + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_commandCommonArgs(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, + sgp, + self._json_keys, + 'corpus', + instance='instance', + bucket='bucket') + session.Initialize() + call_args = cmd_mock.call_args[0][0] + self.assertIn('imgtest', call_args) + self.assertIn('init', call_args) + self.assertIn('--passfail', call_args) + assertArgWith(self, call_args, '--instance', 'instance') + assertArgWith(self, call_args, '--bucket', 'bucket') + assertArgWith(self, call_args, '--corpus', 'corpus') + # The keys file should have been copied to the working directory. + assertArgWith(self, call_args, '--keys-file', + os.path.join(self._working_dir, 'gold_keys.json')) + assertArgWith(self, call_args, '--work-dir', self._working_dir) + assertArgWith(self, call_args, '--failure-file', session._triage_link_file) + assertArgWith(self, call_args, '--commit', 'a') + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_commandTryjobArgs(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a', + gerrit_issue=1, + gerrit_patchset=2, + buildbucket_id=3) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + session.Initialize() + call_args = cmd_mock.call_args[0][0] + assertArgWith(self, call_args, '--issue', '1') + assertArgWith(self, call_args, '--patchset', '2') + assertArgWith(self, call_args, '--jobid', '3') + assertArgWith(self, call_args, '--crs', 'gerrit') + assertArgWith(self, call_args, '--cis', 'buildbucket') + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_commandTryjobArgsNonDefaultCrs(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(code_review_system='foo', + git_revision='a', + gerrit_issue=1, + gerrit_patchset=2, + buildbucket_id=3) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + session.Initialize() + call_args = cmd_mock.call_args[0][0] + assertArgWith(self, call_args, '--issue', '1') + assertArgWith(self, call_args, '--patchset', '2') + assertArgWith(self, call_args, '--jobid', '3') + assertArgWith(self, call_args, '--crs', 'foo') + assertArgWith(self, call_args, '--cis', 'buildbucket') + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_commandTryjobArgsMissing(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + session.Initialize() + call_args = cmd_mock.call_args[0][0] + self.assertNotIn('--issue', call_args) + self.assertNotIn('--patchset', call_args) + self.assertNotIn('--jobid', call_args) + self.assertNotIn('--crs', call_args) + self.assertNotIn('--cis', call_args) + + +class SkiaGoldSessionCompareTest(fake_filesystem_unittest.TestCase): + """Tests the functionality of SkiaGoldSession.Compare.""" + + def setUp(self): + self.setUpPyfakefs() + self._working_dir = tempfile.mkdtemp() + self._json_keys = tempfile.NamedTemporaryFile(delete=False).name + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_commandOutputReturned(self, cmd_mock): + cmd_mock.return_value = (1, 'Something bad :(') + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + rc, stdout = session.Compare(None, None) + self.assertEqual(cmd_mock.call_count, 1) + self.assertEqual(rc, 1) + self.assertEqual(stdout, 'Something bad :(') + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_bypassSkiaGoldFunctionality(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a', + bypass_skia_gold_functionality=True) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + rc, _ = session.Compare(None, None) + self.assertEqual(rc, 0) + cmd_mock.assert_not_called() + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_commandWithLocalPixelTestsTrue(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=True) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + session.Compare(None, None) + self.assertIn('--dryrun', cmd_mock.call_args[0][0]) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_commandWithForceDryrunTrue(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + session.Compare(None, None, force_dryrun=True) + self.assertIn('--dryrun', cmd_mock.call_args[0][0]) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_commandWithLocalPixelTestsFalse(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=False) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + session.Compare(None, None) + self.assertNotIn('--dryrun', cmd_mock.call_args[0][0]) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_commandWithInexactArgs(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + session.Compare(None, None, inexact_matching_args=['--inexact', 'foobar']) + self.assertIn('--inexact', cmd_mock.call_args[0][0]) + self.assertIn('foobar', cmd_mock.call_args[0][0]) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_commandCommonArgs(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, + sgp, + self._json_keys, + 'corpus', + instance='instance') + session.Compare('name', 'png_file') + call_args = cmd_mock.call_args[0][0] + self.assertIn('imgtest', call_args) + self.assertIn('add', call_args) + assertArgWith(self, call_args, '--test-name', 'name') + assertArgWith(self, call_args, '--png-file', 'png_file') + assertArgWith(self, call_args, '--work-dir', self._working_dir) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_noLinkOnSuccess(self, cmd_mock): + cmd_mock.return_value = (0, None) + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + rc, _ = session.Compare('name', 'png_file') + self.assertEqual(rc, 0) + comparison_result = session._comparison_results['name'] + self.assertEqual(comparison_result.public_triage_link, None) + self.assertEqual(comparison_result.internal_triage_link, None) + self.assertNotEqual(comparison_result.triage_link_omission_reason, None) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_clLinkOnTrybot(self, cmd_mock): + cmd_mock.return_value = (1, None) + args = createSkiaGoldArgs(git_revision='a', + gerrit_issue=1, + gerrit_patchset=2, + buildbucket_id=3) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, + sgp, + self._json_keys, + None, + instance='instance') + rc, _ = session.Compare('name', 'png_file') + self.assertEqual(rc, 1) + comparison_result = session._comparison_results['name'] + self.assertNotEqual(comparison_result.public_triage_link, None) + self.assertNotEqual(comparison_result.internal_triage_link, None) + internal_link = 'https://instance-gold.skia.org/cl/gerrit/1' + public_link = 'https://instance-public-gold.skia.org/cl/gerrit/1' + self.assertEqual(comparison_result.internal_triage_link, internal_link) + self.assertEqual(comparison_result.public_triage_link, public_link) + self.assertEqual(comparison_result.triage_link_omission_reason, None) + self.assertEqual(session.GetTriageLinks('name'), + (public_link, internal_link)) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_individualLinkOnCi(self, cmd_mock): + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, + sgp, + self._json_keys, + None, + instance='foobar') + + internal_link = 'foobar-gold.skia.org' + public_link = 'foobar-public-gold.skia.org' + + def WriteTriageLinkFile(_): + with open(session._triage_link_file, 'w') as f: + f.write(internal_link) + return (1, None) + + cmd_mock.side_effect = WriteTriageLinkFile + rc, _ = session.Compare('name', 'png_file') + self.assertEqual(rc, 1) + comparison_result = session._comparison_results['name'] + self.assertNotEqual(comparison_result.public_triage_link, None) + self.assertNotEqual(comparison_result.internal_triage_link, None) + self.assertEqual(comparison_result.internal_triage_link, internal_link) + self.assertEqual(comparison_result.public_triage_link, public_link) + self.assertEqual(comparison_result.triage_link_omission_reason, None) + self.assertEqual(session.GetTriageLinks('name'), + (public_link, internal_link)) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_validOmissionOnMissingLink(self, cmd_mock): + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + + def WriteTriageLinkFile(_): + with open(session._triage_link_file, 'w'): + pass + return (1, None) + + cmd_mock.side_effect = WriteTriageLinkFile + rc, _ = session.Compare('name', 'png_file') + self.assertEqual(rc, 1) + comparison_result = session._comparison_results['name'] + self.assertEqual(comparison_result.public_triage_link, None) + self.assertEqual(comparison_result.internal_triage_link, None) + self.assertIn('Gold did not provide a triage link', + comparison_result.triage_link_omission_reason) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_validOmissionOnIoError(self, cmd_mock): + cmd_mock.return_value = (1, None) + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + + def DeleteTriageLinkFile(_): + os.remove(session._triage_link_file) + return (1, None) + + cmd_mock.side_effect = DeleteTriageLinkFile + rc, _ = session.Compare('name', 'png_file') + self.assertEqual(rc, 1) + comparison_result = session._comparison_results['name'] + self.assertEqual(comparison_result.public_triage_link, None) + self.assertEqual(comparison_result.internal_triage_link, None) + self.assertNotEqual(comparison_result.triage_link_omission_reason, None) + self.assertIn('Failed to read', + comparison_result.triage_link_omission_reason) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_optionalKeysPassedToGoldctl(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=True) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + session.Compare(None, None, optional_keys={'foo': 'bar'}) + assertArgWith(self, cmd_mock.call_args[0][0], '--add-test-optional-key', + 'foo:bar') + + +class SkiaGoldSessionDiffTest(fake_filesystem_unittest.TestCase): + """Tests the functionality of SkiaGoldSession.Diff.""" + + def setUp(self): + self.setUpPyfakefs() + self._working_dir = tempfile.mkdtemp() + self._json_keys = tempfile.NamedTemporaryFile(delete=False).name + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_StoreDiffLinks') + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_commandOutputReturned(self, cmd_mock, _): + cmd_mock.return_value = (1, 'Something bad :(') + args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=False) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + rc, stdout = session.Diff(None, None, None) + self.assertEqual(cmd_mock.call_count, 1) + self.assertEqual(rc, 1) + self.assertEqual(stdout, 'Something bad :(') + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_bypassSkiaGoldFunctionality(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a', + bypass_skia_gold_functionality=True) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + self._json_keys, None, None) + with self.assertRaises(RuntimeError): + session.Diff(None, None, None) + + +class SkiaGoldSessionTriageLinkOmissionTest(fake_filesystem_unittest.TestCase): + """Tests the functionality of SkiaGoldSession.GetTriageLinkOmissionReason.""" + + def setUp(self): + self.setUpPyfakefs() + self._working_dir = tempfile.mkdtemp() + + def _CreateSession(self): + sgp = skia_gold_properties.SkiaGoldProperties(createSkiaGoldArgs()) + json_keys = tempfile.NamedTemporaryFile(delete=False).name + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + json_keys, None, None) + session._comparison_results = { + 'foo': skia_gold_session.SkiaGoldSession.ComparisonResults(), + } + return session + + def test_noComparison(self): + session = self._CreateSession() + session._comparison_results = {} + reason = session.GetTriageLinkOmissionReason('foo') + self.assertEqual(reason, 'No image comparison performed for foo') + + def test_validReason(self): + session = self._CreateSession() + session._comparison_results['foo'].triage_link_omission_reason = 'bar' + reason = session.GetTriageLinkOmissionReason('foo') + self.assertEqual(reason, 'bar') + + def test_onlyLocal(self): + session = self._CreateSession() + session._comparison_results['foo'].local_diff_given_image = 'bar' + reason = session.GetTriageLinkOmissionReason('foo') + self.assertEqual(reason, 'Gold only used to do a local image diff') + + def test_onlyWithoutTriageLink(self): + session = self._CreateSession() + comparison_result = session._comparison_results['foo'] + comparison_result.public_triage_link = 'bar' + with self.assertRaises(AssertionError): + session.GetTriageLinkOmissionReason('foo') + comparison_result.public_triage_link = None + comparison_result.internal_triage_link = 'bar' + with self.assertRaises(AssertionError): + session.GetTriageLinkOmissionReason('foo') + + def test_resultsShouldNotExist(self): + session = self._CreateSession() + with self.assertRaises(RuntimeError): + session.GetTriageLinkOmissionReason('foo') + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/third_party/libwebrtc/build/skia_gold_common/unittest_utils.py b/third_party/libwebrtc/build/skia_gold_common/unittest_utils.py new file mode 100644 index 0000000000..c4f77a14df --- /dev/null +++ b/third_party/libwebrtc/build/skia_gold_common/unittest_utils.py @@ -0,0 +1,36 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Utility methods for Skia Gold functionality unittests.""" + +import collections + +_SkiaGoldArgs = collections.namedtuple('_SkiaGoldArgs', [ + 'local_pixel_tests', + 'no_luci_auth', + 'code_review_system', + 'continuous_integration_system', + 'git_revision', + 'gerrit_issue', + 'gerrit_patchset', + 'buildbucket_id', + 'bypass_skia_gold_functionality', + 'skia_gold_local_png_write_directory', +]) + + +def createSkiaGoldArgs(local_pixel_tests=None, + no_luci_auth=None, + code_review_system=None, + continuous_integration_system=None, + git_revision=None, + gerrit_issue=None, + gerrit_patchset=None, + buildbucket_id=None, + bypass_skia_gold_functionality=None, + skia_gold_local_png_write_directory=None): + return _SkiaGoldArgs(local_pixel_tests, no_luci_auth, code_review_system, + continuous_integration_system, git_revision, + gerrit_issue, gerrit_patchset, buildbucket_id, + bypass_skia_gold_functionality, + skia_gold_local_png_write_directory) |