diff options
Diffstat (limited to '')
50 files changed, 2320 insertions, 0 deletions
diff --git a/testing/mozbase/manifestparser/tests/comment-example.ini b/testing/mozbase/manifestparser/tests/comment-example.ini new file mode 100644 index 0000000000..ba03310e4f --- /dev/null +++ b/testing/mozbase/manifestparser/tests/comment-example.ini @@ -0,0 +1,11 @@ +# See https://bugzilla.mozilla.org/show_bug.cgi?id=813674 + +[test_0180_fileInUse_xp_win_complete.js] +[test_0181_fileInUse_xp_win_partial.js] +[test_0182_rmrfdirFileInUse_xp_win_complete.js] +[test_0183_rmrfdirFileInUse_xp_win_partial.js] +[test_0184_fileInUse_xp_win_complete.js] +[test_0185_fileInUse_xp_win_partial.js] +[test_0186_rmrfdirFileInUse_xp_win_complete.js] +[test_0187_rmrfdirFileInUse_xp_win_partial.js] +# [test_0202_app_launch_apply_update_dirlocked.js] # Test disabled, bug 757632 diff --git a/testing/mozbase/manifestparser/tests/default-skipif.ini b/testing/mozbase/manifestparser/tests/default-skipif.ini new file mode 100644 index 0000000000..d3c2687333 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/default-skipif.ini @@ -0,0 +1,22 @@ +[DEFAULT] +skip-if = os == 'win' && debug # a pesky comment + + +[test1] +skip-if = debug + +[test2] +skip-if = os == 'linux' + +[test3] +skip-if = os == 'win' + +[test4] +skip-if = os == 'win' && debug + +[test5] +foo = bar + +[test6] +skip-if = debug # a second pesky comment + diff --git a/testing/mozbase/manifestparser/tests/default-subsuite.ini b/testing/mozbase/manifestparser/tests/default-subsuite.ini new file mode 100644 index 0000000000..f2c2bbd3b1 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/default-subsuite.ini @@ -0,0 +1,5 @@ +[test1] +subsuite = baz + +[test2] +subsuite = foo diff --git a/testing/mozbase/manifestparser/tests/default-suppfiles.ini b/testing/mozbase/manifestparser/tests/default-suppfiles.ini new file mode 100644 index 0000000000..12af247b82 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/default-suppfiles.ini @@ -0,0 +1,9 @@ +[DEFAULT] +support-files = foo.js # a comment + +[test7] +[test8] +support-files = bar.js # another comment +[test9] +foo = bar + diff --git a/testing/mozbase/manifestparser/tests/filter-example.ini b/testing/mozbase/manifestparser/tests/filter-example.ini new file mode 100644 index 0000000000..13a8734c33 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/filter-example.ini @@ -0,0 +1,11 @@ +# illustrate test filters based on various categories + +[windowstest] +skip-if = os != 'win' + +[fleem] +skip-if = os == 'mac' + +[linuxtest] +skip-if = (os == 'mac') || (os == 'win') +fail-if = toolkit == 'cocoa' diff --git a/testing/mozbase/manifestparser/tests/fleem b/testing/mozbase/manifestparser/tests/fleem new file mode 100644 index 0000000000..744817b823 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/fleem @@ -0,0 +1 @@ +# dummy spot for "fleem" test diff --git a/testing/mozbase/manifestparser/tests/include-example.ini b/testing/mozbase/manifestparser/tests/include-example.ini new file mode 100644 index 0000000000..69e728c3bc --- /dev/null +++ b/testing/mozbase/manifestparser/tests/include-example.ini @@ -0,0 +1,11 @@ +[DEFAULT] +foo = bar + +[include:include/bar.ini] + +[fleem] + +[include:include/foo.ini] +red = roses +blue = violets +yellow = daffodils
\ No newline at end of file diff --git a/testing/mozbase/manifestparser/tests/include-invalid.ini b/testing/mozbase/manifestparser/tests/include-invalid.ini new file mode 100644 index 0000000000..e3ed0dd6b3 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/include-invalid.ini @@ -0,0 +1 @@ +[include:invalid.ini] diff --git a/testing/mozbase/manifestparser/tests/include/bar.ini b/testing/mozbase/manifestparser/tests/include/bar.ini new file mode 100644 index 0000000000..bcb312d1db --- /dev/null +++ b/testing/mozbase/manifestparser/tests/include/bar.ini @@ -0,0 +1,4 @@ +[DEFAULT] +foo = fleem + +[crash-handling]
\ No newline at end of file diff --git a/testing/mozbase/manifestparser/tests/include/crash-handling b/testing/mozbase/manifestparser/tests/include/crash-handling new file mode 100644 index 0000000000..8e19a63751 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/include/crash-handling @@ -0,0 +1 @@ +# dummy spot for "crash-handling" test diff --git a/testing/mozbase/manifestparser/tests/include/flowers b/testing/mozbase/manifestparser/tests/include/flowers new file mode 100644 index 0000000000..a25acfbe21 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/include/flowers @@ -0,0 +1 @@ +# dummy spot for "flowers" test diff --git a/testing/mozbase/manifestparser/tests/include/foo.ini b/testing/mozbase/manifestparser/tests/include/foo.ini new file mode 100644 index 0000000000..cfc90ace83 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/include/foo.ini @@ -0,0 +1,5 @@ +[DEFAULT] +blue = ocean + +[flowers] +yellow = submarine
\ No newline at end of file diff --git a/testing/mozbase/manifestparser/tests/just-defaults.ini b/testing/mozbase/manifestparser/tests/just-defaults.ini new file mode 100644 index 0000000000..83a0cec0c6 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/just-defaults.ini @@ -0,0 +1,2 @@ +[DEFAULT] +foo = bar diff --git a/testing/mozbase/manifestparser/tests/manifest.ini b/testing/mozbase/manifestparser/tests/manifest.ini new file mode 100644 index 0000000000..cd8852134c --- /dev/null +++ b/testing/mozbase/manifestparser/tests/manifest.ini @@ -0,0 +1,13 @@ +[DEFAULT] +subsuite = mozbase +[test_expressionparser.py] +[test_manifestparser.py] +[test_testmanifest.py] +[test_read_ini.py] +[test_convert_directory.py] +[test_filters.py] +[test_chunking.py] +[test_convert_symlinks.py] +disabled = https://bugzilla.mozilla.org/show_bug.cgi?id=920938 +[test_default_overrides.py] +[test_util.py] diff --git a/testing/mozbase/manifestparser/tests/missing-path.ini b/testing/mozbase/manifestparser/tests/missing-path.ini new file mode 100644 index 0000000000..919d8e04da --- /dev/null +++ b/testing/mozbase/manifestparser/tests/missing-path.ini @@ -0,0 +1,2 @@ +[foo] +[bar] diff --git a/testing/mozbase/manifestparser/tests/mozmill-example.ini b/testing/mozbase/manifestparser/tests/mozmill-example.ini new file mode 100644 index 0000000000..114cf48c4b --- /dev/null +++ b/testing/mozbase/manifestparser/tests/mozmill-example.ini @@ -0,0 +1,80 @@ +[testAddons/testDisableEnablePlugin.js] +[testAddons/testGetAddons.js] +[testAddons/testSearchAddons.js] +[testAwesomeBar/testAccessLocationBar.js] +[testAwesomeBar/testCheckItemHighlight.js] +[testAwesomeBar/testEscapeAutocomplete.js] +[testAwesomeBar/testFaviconInAutocomplete.js] +[testAwesomeBar/testGoButton.js] +[testAwesomeBar/testLocationBarSearches.js] +[testAwesomeBar/testPasteLocationBar.js] +[testAwesomeBar/testSuggestHistoryBookmarks.js] +[testAwesomeBar/testVisibleItemsMax.js] +[testBookmarks/testAddBookmarkToMenu.js] +[testCookies/testDisableCookies.js] +[testCookies/testEnableCookies.js] +[testCookies/testRemoveAllCookies.js] +[testCookies/testRemoveCookie.js] +[testDownloading/testCloseDownloadManager.js] +[testDownloading/testDownloadStates.js] +[testDownloading/testOpenDownloadManager.js] +[testFindInPage/testFindInPage.js] +[testFormManager/testAutoCompleteOff.js] +[testFormManager/testBasicFormCompletion.js] +[testFormManager/testClearFormHistory.js] +[testFormManager/testDisableFormManager.js] +[testGeneral/testGoogleSuggestions.js] +[testGeneral/testStopReloadButtons.js] +[testInstallation/testBreakpadInstalled.js] +[testLayout/testNavigateFTP.js] +[testPasswordManager/testPasswordNotSaved.js] +[testPasswordManager/testPasswordSavedAndDeleted.js] +[testPopups/testPopupsAllowed.js] +[testPopups/testPopupsBlocked.js] +[testPreferences/testPaneRetention.js] +[testPreferences/testPreferredLanguage.js] +[testPreferences/testRestoreHomepageToDefault.js] +[testPreferences/testSetToCurrentPage.js] +[testPreferences/testSwitchPanes.js] +[testPrivateBrowsing/testAboutPrivateBrowsing.js] +[testPrivateBrowsing/testCloseWindow.js] +[testPrivateBrowsing/testDisabledElements.js] +[testPrivateBrowsing/testDisabledPermissions.js] +[testPrivateBrowsing/testDownloadManagerClosed.js] +[testPrivateBrowsing/testGeolocation.js] +[testPrivateBrowsing/testStartStopPBMode.js] +[testPrivateBrowsing/testTabRestoration.js] +[testPrivateBrowsing/testTabsDismissedOnStop.js] +[testSearch/testAddMozSearchProvider.js] +[testSearch/testFocusAndSearch.js] +[testSearch/testGetMoreSearchEngines.js] +[testSearch/testOpenSearchAutodiscovery.js] +[testSearch/testRemoveSearchEngine.js] +[testSearch/testReorderSearchEngines.js] +[testSearch/testRestoreDefaults.js] +[testSearch/testSearchSelection.js] +[testSearch/testSearchSuggestions.js] +[testSecurity/testBlueLarry.js] +[testSecurity/testDefaultPhishingEnabled.js] +[testSecurity/testDefaultSecurityPrefs.js] +[testSecurity/testEncryptedPageWarning.js] +[testSecurity/testGreenLarry.js] +[testSecurity/testGreyLarry.js] +[testSecurity/testIdentityPopupOpenClose.js] +[testSecurity/testSSLDisabledErrorPage.js] +[testSecurity/testSafeBrowsingNotificationBar.js] +[testSecurity/testSafeBrowsingWarningPages.js] +[testSecurity/testSecurityInfoViaMoreInformation.js] +[testSecurity/testSecurityNotification.js] +[testSecurity/testSubmitUnencryptedInfoWarning.js] +[testSecurity/testUnknownIssuer.js] +[testSecurity/testUntrustedConnectionErrorPage.js] +[testSessionStore/testUndoTabFromContextMenu.js] +[testTabbedBrowsing/testBackgroundTabScrolling.js] +[testTabbedBrowsing/testCloseTab.js] +[testTabbedBrowsing/testNewTab.js] +[testTabbedBrowsing/testNewWindow.js] +[testTabbedBrowsing/testOpenInBackground.js] +[testTabbedBrowsing/testOpenInForeground.js] +[testTechnicalTools/testAccessPageInfoDialog.js] +[testToolbar/testBackForwardButtons.js] diff --git a/testing/mozbase/manifestparser/tests/mozmill-restart-example.ini b/testing/mozbase/manifestparser/tests/mozmill-restart-example.ini new file mode 100644 index 0000000000..2ce30e7c6a --- /dev/null +++ b/testing/mozbase/manifestparser/tests/mozmill-restart-example.ini @@ -0,0 +1,26 @@ +[DEFAULT] +type = restart + +[restartTests/testExtensionInstallUninstall/test2.js] +foo = bar + +[restartTests/testExtensionInstallUninstall/test1.js] +foo = baz + +[restartTests/testExtensionInstallUninstall/test3.js] +[restartTests/testSoftwareUpdateAutoProxy/test2.js] +[restartTests/testSoftwareUpdateAutoProxy/test1.js] +[restartTests/testPrimaryPassword/test1.js] +[restartTests/testExtensionInstallGetAddons/test2.js] +[restartTests/testExtensionInstallGetAddons/test1.js] +[restartTests/testMultipleExtensionInstallation/test2.js] +[restartTests/testMultipleExtensionInstallation/test1.js] +[restartTests/testThemeInstallUninstall/test2.js] +[restartTests/testThemeInstallUninstall/test1.js] +[restartTests/testThemeInstallUninstall/test3.js] +[restartTests/testDefaultBookmarks/test1.js] +[softwareUpdate/testFallbackUpdate/test2.js] +[softwareUpdate/testFallbackUpdate/test1.js] +[softwareUpdate/testFallbackUpdate/test3.js] +[softwareUpdate/testDirectUpdate/test2.js] +[softwareUpdate/testDirectUpdate/test1.js] diff --git a/testing/mozbase/manifestparser/tests/no-tests.ini b/testing/mozbase/manifestparser/tests/no-tests.ini new file mode 100644 index 0000000000..83a0cec0c6 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/no-tests.ini @@ -0,0 +1,2 @@ +[DEFAULT] +foo = bar diff --git a/testing/mozbase/manifestparser/tests/parent/include/first/manifest.ini b/testing/mozbase/manifestparser/tests/parent/include/first/manifest.ini new file mode 100644 index 0000000000..828525c18f --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/include/first/manifest.ini @@ -0,0 +1,3 @@ +[parent:../manifest.ini] + +[testFirst.js] diff --git a/testing/mozbase/manifestparser/tests/parent/include/manifest.ini b/testing/mozbase/manifestparser/tests/parent/include/manifest.ini new file mode 100644 index 0000000000..fb9756d6af --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/include/manifest.ini @@ -0,0 +1,8 @@ +[DEFAULT] +top = data + +[include:first/manifest.ini] +disabled = YES + +[include:second/manifest.ini] +disabled = NO diff --git a/testing/mozbase/manifestparser/tests/parent/include/second/manifest.ini b/testing/mozbase/manifestparser/tests/parent/include/second/manifest.ini new file mode 100644 index 0000000000..31f0537566 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/include/second/manifest.ini @@ -0,0 +1,3 @@ +[parent:../manifest.ini] + +[testSecond.js] diff --git a/testing/mozbase/manifestparser/tests/parent/level_1/level_1.ini b/testing/mozbase/manifestparser/tests/parent/level_1/level_1.ini new file mode 100644 index 0000000000..ac7c370c3e --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/level_1/level_1.ini @@ -0,0 +1,5 @@ +[DEFAULT] +x = level_1 + +[test_1] +[test_2] diff --git a/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_2.ini b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_2.ini new file mode 100644 index 0000000000..ada6a510d7 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_2.ini @@ -0,0 +1,3 @@ +[parent:../level_1.ini] + +[test_2] diff --git a/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/level_3.ini b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/level_3.ini new file mode 100644 index 0000000000..2edd647fcc --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/level_3.ini @@ -0,0 +1,3 @@ +[parent:../level_2.ini] + +[test_3] diff --git a/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/level_3_default.ini b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/level_3_default.ini new file mode 100644 index 0000000000..d6aae60ae1 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/level_3_default.ini @@ -0,0 +1,6 @@ +[parent:../level_2.ini] + +[DEFAULT] +x = level_3 + +[test_3] diff --git a/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/test_3 b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/test_3 new file mode 100644 index 0000000000..f5de587529 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/test_3 @@ -0,0 +1 @@ +# dummy spot for "test_3" test diff --git a/testing/mozbase/manifestparser/tests/parent/level_1/level_2/test_2 b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/test_2 new file mode 100644 index 0000000000..5b77e04f31 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/test_2 @@ -0,0 +1 @@ +# dummy spot for "test_2" test diff --git a/testing/mozbase/manifestparser/tests/parent/level_1/test_1 b/testing/mozbase/manifestparser/tests/parent/level_1/test_1 new file mode 100644 index 0000000000..dccbf04e4d --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/level_1/test_1 @@ -0,0 +1 @@ +# dummy spot for "test_1" test diff --git a/testing/mozbase/manifestparser/tests/parent/root/dummy b/testing/mozbase/manifestparser/tests/parent/root/dummy new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/root/dummy diff --git a/testing/mozbase/manifestparser/tests/path-example.ini b/testing/mozbase/manifestparser/tests/path-example.ini new file mode 100644 index 0000000000..366782d95d --- /dev/null +++ b/testing/mozbase/manifestparser/tests/path-example.ini @@ -0,0 +1,2 @@ +[foo] +path = fleem
\ No newline at end of file diff --git a/testing/mozbase/manifestparser/tests/relative-path.ini b/testing/mozbase/manifestparser/tests/relative-path.ini new file mode 100644 index 0000000000..57105489ba --- /dev/null +++ b/testing/mozbase/manifestparser/tests/relative-path.ini @@ -0,0 +1,5 @@ +[foo] +path = ../fleem + +[bar] +path = ../testsSIBLING/example diff --git a/testing/mozbase/manifestparser/tests/subsuite.ini b/testing/mozbase/manifestparser/tests/subsuite.ini new file mode 100644 index 0000000000..c1a70bd44e --- /dev/null +++ b/testing/mozbase/manifestparser/tests/subsuite.ini @@ -0,0 +1,13 @@ +[test1] +subsuite=bar,foo=="bar" # this has a comment + +[test2] +subsuite=bar,foo=="bar" + +[test3] +subsuite=baz + +[test4] +[test5] +[test6] +subsuite=bar,foo=="szy" || foo=="bar"
\ No newline at end of file diff --git a/testing/mozbase/manifestparser/tests/test_chunking.py b/testing/mozbase/manifestparser/tests/test_chunking.py new file mode 100644 index 0000000000..3c649be99f --- /dev/null +++ b/testing/mozbase/manifestparser/tests/test_chunking.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python + +import os +import random +from collections import defaultdict +from itertools import chain +from unittest import TestCase + +import mozunit +from manifestparser.filters import chunk_by_dir, chunk_by_runtime, chunk_by_slice +from six import iteritems +from six.moves import range + +here = os.path.dirname(os.path.abspath(__file__)) + + +class ChunkBySlice(TestCase): + """Test chunking related filters""" + + def generate_tests(self, num, disabled=None): + disabled = disabled or [] + tests = [] + for i in range(num): + test = {"name": "test%i" % i} + if i in disabled: + test["disabled"] = "" + tests.append(test) + return tests + + def run_all_combos(self, num_tests, disabled=None): + tests = self.generate_tests(num_tests, disabled=disabled) + + for total in range(1, num_tests + 1): + res = [] + res_disabled = [] + for chunk in range(1, total + 1): + f = chunk_by_slice(chunk, total) + res.append(list(f(tests, {}))) + if disabled: + f.disabled = True + res_disabled.append(list(f(tests, {}))) + + lengths = [len([t for t in c if "disabled" not in t]) for c in res] + # the chunk with the most tests should have at most one more test + # than the chunk with the least tests + self.assertLessEqual(max(lengths) - min(lengths), 1) + + # chaining all chunks back together should equal the original list + # of tests + self.assertEqual(list(chain.from_iterable(res)), list(tests)) + + if disabled: + lengths = [len(c) for c in res_disabled] + self.assertLessEqual(max(lengths) - min(lengths), 1) + self.assertEqual(list(chain.from_iterable(res_disabled)), list(tests)) + + def test_chunk_by_slice(self): + chunk = chunk_by_slice(1, 1) + self.assertEqual(list(chunk([], {})), []) + + self.run_all_combos(num_tests=1) + self.run_all_combos(num_tests=10, disabled=[1, 2]) + + num_tests = 67 + disabled = list(i for i in range(num_tests) if i % 4 == 0) + self.run_all_combos(num_tests=num_tests, disabled=disabled) + + def test_two_times_more_chunks_than_tests(self): + # test case for bug 1182817 + tests = self.generate_tests(5) + + total_chunks = 10 + for i in range(1, total_chunks + 1): + # ensure IndexError is not raised + chunk_by_slice(i, total_chunks)(tests, {}) + + +class ChunkByDir(TestCase): + """Test chunking related filters""" + + def generate_tests(self, dirs): + """ + :param dirs: dict of the form, + { <dir>: <num tests> } + """ + i = 0 + for d, num in iteritems(dirs): + for _ in range(num): + i += 1 + name = "test%i" % i + test = {"name": name, "relpath": os.path.join(d, name)} + yield test + + def run_all_combos(self, dirs): + tests = list(self.generate_tests(dirs)) + + deepest = max(len(t["relpath"].split(os.sep)) - 1 for t in tests) + for depth in range(1, deepest + 1): + + def num_groups(tests): + unique = set() + for p in [t["relpath"] for t in tests]: + p = p.split(os.sep) + p = p[: min(depth, len(p) - 1)] + unique.add(os.sep.join(p)) + return len(unique) + + for total in range(1, num_groups(tests) + 1): + res = [] + for this in range(1, total + 1): + f = chunk_by_dir(this, total, depth) + res.append(list(f(tests, {}))) + + lengths = list(map(num_groups, res)) + # the chunk with the most dirs should have at most one more + # dir than the chunk with the least dirs + self.assertLessEqual(max(lengths) - min(lengths), 1) + + all_chunks = list(chain.from_iterable(res)) + # chunk_by_dir will mess up order, but chained chunks should + # contain all of the original tests and be the same length + self.assertEqual(len(all_chunks), len(tests)) + for t in tests: + self.assertIn(t, all_chunks) + + def test_chunk_by_dir(self): + chunk = chunk_by_dir(1, 1, 1) + self.assertEqual(list(chunk([], {})), []) + + dirs = { + "a": 2, + } + self.run_all_combos(dirs) + + dirs = { + "": 1, + "foo": 1, + "bar": 0, + "/foobar": 1, + } + self.run_all_combos(dirs) + + dirs = { + "a": 1, + "b": 1, + "a/b": 2, + "a/c": 1, + } + self.run_all_combos(dirs) + + dirs = { + "a": 5, + "a/b": 4, + "a/b/c": 7, + "a/b/c/d": 1, + "a/b/c/e": 3, + "b/c": 2, + "b/d": 5, + "b/d/e": 6, + "c": 8, + "c/d/e/f/g/h/i/j/k/l": 5, + "c/d/e/f/g/i/j/k/l/m/n": 2, + "c/e": 1, + } + self.run_all_combos(dirs) + + +class ChunkByRuntime(TestCase): + """Test chunking related filters""" + + def generate_tests(self, dirs): + """ + :param dirs: dict of the form, + { <dir>: <num tests> } + """ + i = 0 + for d, num in iteritems(dirs): + for _ in range(num): + i += 1 + name = "test%i" % i + manifest = os.path.join(d, "manifest.ini") + test = { + "name": name, + "relpath": os.path.join(d, name), + "manifest": manifest, + "manifest_relpath": manifest, + } + yield test + + def get_runtimes(self, tests): + runtimes = defaultdict(int) + for test in tests: + runtimes[test["manifest_relpath"]] += random.randint(0, 100) + return runtimes + + def chunk_by_round_robin(self, tests, total, runtimes): + tests_by_manifest = [] + for manifest, runtime in iteritems(runtimes): + mtests = [t for t in tests if t["manifest_relpath"] == manifest] + tests_by_manifest.append((runtime, mtests)) + tests_by_manifest.sort(key=lambda x: x[0], reverse=False) + + chunks = [[] for i in range(total)] + d = 1 # direction + i = 0 + for runtime, batch in tests_by_manifest: + chunks[i].extend(batch) + + # "draft" style (last pick goes first in the next round) + if (i == 0 and d == -1) or (i == total - 1 and d == 1): + d = -d + else: + i += d + + # make sure this test algorithm is valid + all_chunks = list(chain.from_iterable(chunks)) + self.assertEqual(len(all_chunks), len(tests)) + for t in tests: + self.assertIn(t, all_chunks) + return chunks + + def run_all_combos(self, dirs): + tests = list(self.generate_tests(dirs)) + runtimes = self.get_runtimes(tests) + + for total in range(1, len(dirs) + 1): + chunks = [] + for this in range(1, total + 1): + f = chunk_by_runtime(this, total, runtimes) + ret = list(f(tests, {})) + chunks.append(ret) + + # chunk_by_runtime will mess up order, but chained chunks should + # contain all of the original tests and be the same length + all_chunks = list(chain.from_iterable(chunks)) + self.assertEqual(len(all_chunks), len(tests)) + for t in tests: + self.assertIn(t, all_chunks) + + # calculate delta between slowest and fastest chunks + def runtime_delta(chunks): + totals = [] + for chunk in chunks: + manifests = set([t["manifest_relpath"] for t in chunk]) + total = sum(runtimes[m] for m in manifests) + totals.append(total) + return max(totals) - min(totals) + + delta = runtime_delta(chunks) + + # redo the chunking a second time using a round robin style + # algorithm + chunks = self.chunk_by_round_robin(tests, total, runtimes) + # sanity check the round robin algorithm + all_chunks = list(chain.from_iterable(chunks)) + self.assertEqual(len(all_chunks), len(tests)) + for t in tests: + self.assertIn(t, all_chunks) + + # since chunks will never have exactly equal runtimes, it's hard + # to tell if they were chunked optimally. Make sure it at least + # beats a naive round robin approach. + self.assertLessEqual(delta, runtime_delta(chunks)) + + def test_chunk_by_runtime(self): + random.seed(42) + + chunk = chunk_by_runtime(1, 1, {}) + self.assertEqual(list(chunk([], {})), []) + + dirs = { + "a": 2, + } + self.run_all_combos(dirs) + + dirs = { + "": 1, + "foo": 1, + "bar": 0, + "/foobar": 1, + } + self.run_all_combos(dirs) + + dirs = { + "a": 1, + "b": 1, + "a/b": 2, + "a/c": 1, + } + self.run_all_combos(dirs) + + dirs = { + "a": 5, + "a/b": 4, + "a/b/c": 7, + "a/b/c/d": 1, + "a/b/c/e": 3, + "b/c": 2, + "b/d": 5, + "b/d/e": 6, + "c": 8, + "c/d/e/f/g/h/i/j/k/l": 5, + "c/d/e/f/g/i/j/k/l/m/n": 2, + "c/e": 1, + } + self.run_all_combos(dirs) + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/mozbase/manifestparser/tests/test_convert_directory.py b/testing/mozbase/manifestparser/tests/test_convert_directory.py new file mode 100755 index 0000000000..9656b9c72b --- /dev/null +++ b/testing/mozbase/manifestparser/tests/test_convert_directory.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import shutil +import tempfile +import unittest + +import mozunit +from manifestparser import ManifestParser, convert + +here = os.path.dirname(os.path.abspath(__file__)) + +# In some cases tempfile.mkdtemp() may returns a path which contains +# symlinks. Some tests here will then break, as the manifestparser.convert +# function returns paths that does not contains symlinks. +# +# Workaround is to use the following function, if absolute path of temp dir +# must be compared. + + +def create_realpath_tempdir(): + """ + Create a tempdir without symlinks. + """ + return os.path.realpath(tempfile.mkdtemp()) + + +class TestDirectoryConversion(unittest.TestCase): + """test conversion of a directory tree to a manifest structure""" + + def create_stub(self, directory=None): + """stub out a directory with files in it""" + + files = ("foo", "bar", "fleem") + if directory is None: + directory = create_realpath_tempdir() + for i in files: + open(os.path.join(directory, i), "w").write(i) + subdir = os.path.join(directory, "subdir") + os.mkdir(subdir) + open(os.path.join(subdir, "subfile"), "w").write("baz") + return directory + + def test_directory_to_manifest(self): + """ + Test our ability to convert a static directory structure to a + manifest. + """ + + # create a stub directory + stub = self.create_stub() + try: + stub = stub.replace(os.path.sep, "/") + self.assertTrue(os.path.exists(stub) and os.path.isdir(stub)) + + # Make a manifest for it + manifest = convert([stub]) + out_tmpl = """[%(stub)s/bar] + +[%(stub)s/fleem] + +[%(stub)s/foo] + +[%(stub)s/subdir/subfile] + +""" # noqa + self.assertEqual(str(manifest), out_tmpl % dict(stub=stub)) + except BaseException: + raise + finally: + shutil.rmtree(stub) # cleanup + + def test_convert_directory_manifests_in_place(self): + """ + keep the manifests in place + """ + + stub = self.create_stub() + try: + ManifestParser.populate_directory_manifests([stub], filename="manifest.ini") + self.assertEqual( + sorted(os.listdir(stub)), + ["bar", "fleem", "foo", "manifest.ini", "subdir"], + ) + parser = ManifestParser() + parser.read(os.path.join(stub, "manifest.ini")) + self.assertEqual( + [i["name"] for i in parser.tests], ["subfile", "bar", "fleem", "foo"] + ) + parser = ManifestParser() + parser.read(os.path.join(stub, "subdir", "manifest.ini")) + self.assertEqual(len(parser.tests), 1) + self.assertEqual(parser.tests[0]["name"], "subfile") + except BaseException: + raise + finally: + shutil.rmtree(stub) + + def test_manifest_ignore(self): + """test manifest `ignore` parameter for ignoring directories""" + + stub = self.create_stub() + try: + ManifestParser.populate_directory_manifests( + [stub], filename="manifest.ini", ignore=("subdir",) + ) + parser = ManifestParser() + parser.read(os.path.join(stub, "manifest.ini")) + self.assertEqual([i["name"] for i in parser.tests], ["bar", "fleem", "foo"]) + self.assertFalse( + os.path.exists(os.path.join(stub, "subdir", "manifest.ini")) + ) + except BaseException: + raise + finally: + shutil.rmtree(stub) + + def test_pattern(self): + """test directory -> manifest with a file pattern""" + + stub = self.create_stub() + try: + parser = convert([stub], pattern="f*", relative_to=stub) + self.assertEqual([i["name"] for i in parser.tests], ["fleem", "foo"]) + + # test multiple patterns + parser = convert([stub], pattern=("f*", "s*"), relative_to=stub) + self.assertEqual( + [i["name"] for i in parser.tests], ["fleem", "foo", "subdir/subfile"] + ) + except BaseException: + raise + finally: + shutil.rmtree(stub) + + def test_update(self): + """ + Test our ability to update tests from a manifest and a directory of + files + """ + + # boilerplate + tempdir = create_realpath_tempdir() + for i in range(10): + open(os.path.join(tempdir, str(i)), "w").write(str(i)) + + # otherwise empty directory with a manifest file + newtempdir = create_realpath_tempdir() + manifest_file = os.path.join(newtempdir, "manifest.ini") + manifest_contents = str(convert([tempdir], relative_to=tempdir)) + with open(manifest_file, "w") as f: + f.write(manifest_contents) + + # get the manifest + manifest = ManifestParser(manifests=(manifest_file,)) + + # All of the tests are initially missing: + paths = [str(i) for i in range(10)] + self.assertEqual([i["name"] for i in manifest.missing()], paths) + + # But then we copy one over: + self.assertEqual(manifest.get("name", name="1"), ["1"]) + manifest.update(tempdir, name="1") + self.assertEqual(sorted(os.listdir(newtempdir)), ["1", "manifest.ini"]) + + # Update that one file and copy all the "tests": + open(os.path.join(tempdir, "1"), "w").write("secret door") + manifest.update(tempdir) + self.assertEqual( + sorted(os.listdir(newtempdir)), + ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "manifest.ini"], + ) + self.assertEqual( + open(os.path.join(newtempdir, "1")).read().strip(), "secret door" + ) + + # clean up: + shutil.rmtree(tempdir) + shutil.rmtree(newtempdir) + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/mozbase/manifestparser/tests/test_convert_symlinks.py b/testing/mozbase/manifestparser/tests/test_convert_symlinks.py new file mode 100755 index 0000000000..61054c8b78 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/test_convert_symlinks.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import shutil +import tempfile +import unittest + +import mozunit +from manifestparser import ManifestParser, convert + + +class TestSymlinkConversion(unittest.TestCase): + """ + test conversion of a directory tree with symlinks to a manifest structure + """ + + def create_stub(self, directory=None): + """stub out a directory with files in it""" + + files = ("foo", "bar", "fleem") + if directory is None: + directory = tempfile.mkdtemp() + for i in files: + open(os.path.join(directory, i), "w").write(i) + subdir = os.path.join(directory, "subdir") + os.mkdir(subdir) + open(os.path.join(subdir, "subfile"), "w").write("baz") + return directory + + def test_relpath(self): + """test convert `relative_to` functionality""" + + oldcwd = os.getcwd() + stub = self.create_stub() + try: + # subdir with in-memory manifest + files = ["../bar", "../fleem", "../foo", "subfile"] + subdir = os.path.join(stub, "subdir") + os.chdir(subdir) + parser = convert([stub], relative_to=".") + self.assertEqual([i["name"] for i in parser.tests], files) + except BaseException: + raise + finally: + shutil.rmtree(stub) + os.chdir(oldcwd) + + @unittest.skipIf( + not hasattr(os, "symlink"), "symlinks unavailable on this platform" + ) + def test_relpath_symlink(self): + """ + Ensure `relative_to` works in a symlink. + Not available on windows. + """ + + oldcwd = os.getcwd() + workspace = tempfile.mkdtemp() + try: + tmpdir = os.path.join(workspace, "directory") + os.makedirs(tmpdir) + linkdir = os.path.join(workspace, "link") + os.symlink(tmpdir, linkdir) + self.create_stub(tmpdir) + + # subdir with in-memory manifest + files = ["../bar", "../fleem", "../foo", "subfile"] + subdir = os.path.join(linkdir, "subdir") + os.chdir(os.path.realpath(subdir)) + for directory in (tmpdir, linkdir): + parser = convert([directory], relative_to=".") + self.assertEqual([i["name"] for i in parser.tests], files) + finally: + shutil.rmtree(workspace) + os.chdir(oldcwd) + + # a more complicated example + oldcwd = os.getcwd() + workspace = tempfile.mkdtemp() + try: + tmpdir = os.path.join(workspace, "directory") + os.makedirs(tmpdir) + linkdir = os.path.join(workspace, "link") + os.symlink(tmpdir, linkdir) + self.create_stub(tmpdir) + files = ["../bar", "../fleem", "../foo", "subfile"] + subdir = os.path.join(linkdir, "subdir") + subsubdir = os.path.join(subdir, "sub") + os.makedirs(subsubdir) + linksubdir = os.path.join(linkdir, "linky") + linksubsubdir = os.path.join(subsubdir, "linky") + os.symlink(subdir, linksubdir) + os.symlink(subdir, linksubsubdir) + for dest in (subdir,): + os.chdir(dest) + for directory in (tmpdir, linkdir): + parser = convert([directory], relative_to=".") + self.assertEqual([i["name"] for i in parser.tests], files) + finally: + shutil.rmtree(workspace) + os.chdir(oldcwd) + + @unittest.skipIf( + not hasattr(os, "symlink"), "symlinks unavailable on this platform" + ) + def test_recursion_symlinks(self): + workspace = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, workspace) + + # create two dirs + os.makedirs(os.path.join(workspace, "dir1")) + os.makedirs(os.path.join(workspace, "dir2")) + + # create cyclical symlinks + os.symlink(os.path.join("..", "dir1"), os.path.join(workspace, "dir2", "ldir1")) + os.symlink(os.path.join("..", "dir2"), os.path.join(workspace, "dir1", "ldir2")) + + # create one file in each dir + open(os.path.join(workspace, "dir1", "f1.txt"), "a").close() + open(os.path.join(workspace, "dir1", "ldir2", "f2.txt"), "a").close() + + data = [] + + def callback(rootdirectory, directory, subdirs, files): + for f in files: + data.append(f) + + ManifestParser._walk_directories([workspace], callback) + self.assertEqual(sorted(data), ["f1.txt", "f2.txt"]) + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/mozbase/manifestparser/tests/test_default_overrides.py b/testing/mozbase/manifestparser/tests/test_default_overrides.py new file mode 100755 index 0000000000..bf9c0f7e5d --- /dev/null +++ b/testing/mozbase/manifestparser/tests/test_default_overrides.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import unittest + +import mozunit +from manifestparser import ManifestParser, combine_fields + +here = os.path.dirname(os.path.abspath(__file__)) + + +class TestDefaultSkipif(unittest.TestCase): + """Tests applying a skip-if condition in [DEFAULT] and || with the value for the test""" + + def test_defaults(self): + + default = os.path.join(here, "default-skipif.ini") + parser = ManifestParser(manifests=(default,)) + for test in parser.tests: + if test["name"] == "test1": + self.assertEqual(test["skip-if"], "os == 'win' && debug\ndebug") + elif test["name"] == "test2": + self.assertEqual(test["skip-if"], "os == 'win' && debug\nos == 'linux'") + elif test["name"] == "test3": + self.assertEqual(test["skip-if"], "os == 'win' && debug\nos == 'win'") + elif test["name"] == "test4": + self.assertEqual( + test["skip-if"], "os == 'win' && debug\nos == 'win' && debug" + ) + elif test["name"] == "test5": + self.assertEqual(test["skip-if"], "os == 'win' && debug") + elif test["name"] == "test6": + self.assertEqual(test["skip-if"], "os == 'win' && debug\ndebug") + + +class TestDefaultSupportFiles(unittest.TestCase): + """Tests combining support-files field in [DEFAULT] with the value for a test""" + + def test_defaults(self): + + default = os.path.join(here, "default-suppfiles.ini") + parser = ManifestParser(manifests=(default,)) + expected_supp_files = { + "test7": "foo.js", + "test8": "foo.js bar.js", + "test9": "foo.js", + } + for test in parser.tests: + expected = expected_supp_files[test["name"]] + self.assertEqual(test["support-files"], expected) + + +class TestOmitDefaults(unittest.TestCase): + """Tests passing omit-defaults prevents defaults from propagating to definitions.""" + + def test_defaults(self): + manifests = ( + os.path.join(here, "default-suppfiles.ini"), + os.path.join(here, "default-skipif.ini"), + ) + parser = ManifestParser(manifests=manifests, handle_defaults=False) + expected_supp_files = { + "test8": "bar.js", + } + expected_skip_ifs = { + "test1": "debug", + "test2": "os == 'linux'", + "test3": "os == 'win'", + "test4": "os == 'win' && debug", + "test6": "debug", + } + for test in parser.tests: + for field, expectations in ( + ("support-files", expected_supp_files), + ("skip-if", expected_skip_ifs), + ): + expected = expectations.get(test["name"]) + if not expected: + self.assertNotIn(field, test) + else: + self.assertEqual(test[field], expected) + + expected_defaults = { + os.path.join(here, "default-suppfiles.ini"): { + "support-files": "foo.js", + }, + os.path.join(here, "default-skipif.ini"): { + "skip-if": "os == 'win' && debug", + }, + } + for path, defaults in expected_defaults.items(): + self.assertIn(path, parser.manifest_defaults) + actual_defaults = parser.manifest_defaults[path] + for key, value in defaults.items(): + self.assertIn(key, actual_defaults) + self.assertEqual(value, actual_defaults[key]) + + +class TestSubsuiteDefaults(unittest.TestCase): + """Test that subsuites are handled correctly when managing defaults + outside of the manifest parser.""" + + def test_subsuite_defaults(self): + manifest = os.path.join(here, "default-subsuite.ini") + parser = ManifestParser(manifests=(manifest,), handle_defaults=False) + expected_subsuites = { + "test1": "baz", + "test2": "foo", + } + defaults = parser.manifest_defaults[manifest] + for test in parser.tests: + value = combine_fields(defaults, test) + self.assertEqual(expected_subsuites[value["name"]], value["subsuite"]) + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/mozbase/manifestparser/tests/test_expressionparser.py b/testing/mozbase/manifestparser/tests/test_expressionparser.py new file mode 100755 index 0000000000..806f2347ef --- /dev/null +++ b/testing/mozbase/manifestparser/tests/test_expressionparser.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python + +import unittest + +import mozunit +from manifestparser import parse + + +class ExpressionParserTest(unittest.TestCase): + """Test the conditional expression parser.""" + + def test_basic(self): + + self.assertEqual(parse("1"), 1) + self.assertEqual(parse("100"), 100) + self.assertEqual(parse("true"), True) + self.assertEqual(parse("false"), False) + self.assertEqual("", parse('""')) + self.assertEqual(parse('"foo bar"'), "foo bar") + self.assertEqual(parse("'foo bar'"), "foo bar") + self.assertEqual(parse("foo", foo=1), 1) + self.assertEqual(parse("bar", bar=True), True) + self.assertEqual(parse("abc123", abc123="xyz"), "xyz") + + def test_equality(self): + + self.assertTrue(parse("true == true")) + self.assertTrue(parse("false == false")) + self.assertTrue(parse("1 == 1")) + self.assertTrue(parse("100 == 100")) + self.assertTrue(parse('"some text" == "some text"')) + self.assertTrue(parse("true != false")) + self.assertTrue(parse("1 != 2")) + self.assertTrue(parse('"text" != "other text"')) + self.assertTrue(parse("foo == true", foo=True)) + self.assertTrue(parse("foo == 1", foo=1)) + self.assertTrue(parse('foo == "bar"', foo="bar")) + self.assertTrue(parse("foo == bar", foo=True, bar=True)) + self.assertTrue(parse("true == foo", foo=True)) + self.assertTrue(parse("foo != true", foo=False)) + self.assertTrue(parse("foo != 2", foo=1)) + self.assertTrue(parse('foo != "bar"', foo="abc")) + self.assertTrue(parse("foo != bar", foo=True, bar=False)) + self.assertTrue(parse("true != foo", foo=False)) + self.assertTrue(parse("!false")) + + def test_conjunctures(self): + self.assertTrue(parse("true && true")) + self.assertTrue(parse("true || false")) + self.assertFalse(parse("false || false")) + self.assertFalse(parse("true && false")) + self.assertTrue(parse("true || false && false")) + + def test_parentheses(self): + self.assertTrue(parse("(true)")) + self.assertEqual(parse("(10)"), 10) + self.assertEqual(parse('("foo")'), "foo") + self.assertEqual(parse("(foo)", foo=1), 1) + self.assertTrue(parse("(true == true)"), True) + self.assertTrue(parse("(true != false)")) + self.assertTrue(parse("(true && true)")) + self.assertTrue(parse("(true || false)")) + self.assertTrue(parse("(true && true || false)")) + self.assertFalse(parse("(true || false) && false")) + self.assertTrue(parse("(true || false) && true")) + self.assertTrue(parse("true && (true || false)")) + self.assertTrue(parse("true && (true || false)")) + self.assertTrue(parse("(true && false) || (true && (true || false))")) + + def test_comments(self): + # comments in expressions work accidentally, via an implementation + # detail - the '#' character doesn't match any of the regular + # expressions we specify as tokens, and thus are ignored. + # However, having explicit tests for them means that should the + # implementation ever change, comments continue to work, even if that + # means a new implementation must handle them explicitly. + self.assertTrue(parse("true == true # it does!")) + self.assertTrue(parse("false == false # it does")) + self.assertTrue(parse("false != true # it doesnt")) + self.assertTrue(parse('"string with #" == "string with #" # really, it does')) + self.assertTrue( + parse('"string with #" != "string with # but not the same" # no match!') + ) + + def test_not(self): + """ + Test the ! operator. + """ + self.assertTrue(parse("!false")) + self.assertTrue(parse("!(false)")) + self.assertFalse(parse("!true")) + self.assertFalse(parse("!(true)")) + self.assertTrue(parse("!true || true)")) + self.assertTrue(parse("true || !true)")) + self.assertFalse(parse("!true && true")) + self.assertFalse(parse("true && !true")) + + def test_lesser_than(self): + """ + Test the < operator. + """ + self.assertTrue(parse("1 < 2")) + self.assertFalse(parse("3 < 2")) + self.assertTrue(parse("false || (1 < 2)")) + self.assertTrue(parse("1 < 2 && true")) + self.assertTrue(parse("true && 1 < 2")) + self.assertTrue(parse("!(5 < 1)")) + self.assertTrue(parse("'abc' < 'def'")) + self.assertFalse(parse("1 < 1")) + self.assertFalse(parse("'abc' < 'abc'")) + + def test_greater_than(self): + """ + Test the > operator. + """ + self.assertTrue(parse("2 > 1")) + self.assertFalse(parse("2 > 3")) + self.assertTrue(parse("false || (2 > 1)")) + self.assertTrue(parse("2 > 1 && true")) + self.assertTrue(parse("true && 2 > 1")) + self.assertTrue(parse("!(1 > 5)")) + self.assertTrue(parse("'def' > 'abc'")) + self.assertFalse(parse("1 > 1")) + self.assertFalse(parse("'abc' > 'abc'")) + + def test_lesser_or_equals_than(self): + """ + Test the <= operator. + """ + self.assertTrue(parse("1 <= 2")) + self.assertFalse(parse("3 <= 2")) + self.assertTrue(parse("false || (1 <= 2)")) + self.assertTrue(parse("1 < 2 && true")) + self.assertTrue(parse("true && 1 <= 2")) + self.assertTrue(parse("!(5 <= 1)")) + self.assertTrue(parse("'abc' <= 'def'")) + self.assertTrue(parse("1 <= 1")) + self.assertTrue(parse("'abc' <= 'abc'")) + + def test_greater_or_equals_than(self): + """ + Test the > operator. + """ + self.assertTrue(parse("2 >= 1")) + self.assertFalse(parse("2 >= 3")) + self.assertTrue(parse("false || (2 >= 1)")) + self.assertTrue(parse("2 >= 1 && true")) + self.assertTrue(parse("true && 2 >= 1")) + self.assertTrue(parse("!(1 >= 5)")) + self.assertTrue(parse("'def' >= 'abc'")) + self.assertTrue(parse("1 >= 1")) + self.assertTrue(parse("'abc' >= 'abc'")) + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/mozbase/manifestparser/tests/test_filters.py b/testing/mozbase/manifestparser/tests/test_filters.py new file mode 100644 index 0000000000..5060c7c395 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/test_filters.py @@ -0,0 +1,332 @@ +#!/usr/bin/env python + +import os +from copy import deepcopy +from pprint import pprint + +import mozpack.path as mozpath +import mozunit +import pytest +from manifestparser.filters import ( + enabled, + fail_if, + failures, + filterlist, + pathprefix, + run_if, + skip_if, + subsuite, + tags, +) + +here = os.path.dirname(os.path.abspath(__file__)) + + +def test_data_model(): + def foo(x, y): + return x + + def bar(x, y): + return x + + def baz(x, y): + return x + + fl = filterlist() + + fl.extend([foo, bar]) + assert len(fl) == 2 + assert foo in fl + + fl.append(baz) + assert fl[2] == baz + + fl.remove(baz) + assert baz not in fl + + item = fl.pop() + assert item == bar + + assert fl.index(foo) == 0 + + del fl[0] + assert foo not in fl + with pytest.raises(IndexError): + fl[0] + + +def test_add_non_callable_to_list(): + fl = filterlist() + with pytest.raises(TypeError): + fl.append("foo") + + +def test_add_duplicates_to_list(): + def foo(x, y): + return x + + def bar(x, y): + return x + + sub = subsuite("foo") + fl = filterlist([foo, bar, sub]) + assert len(fl) == 3 + assert fl[0] == foo + + with pytest.raises(ValueError): + fl.append(foo) + + with pytest.raises(ValueError): + fl.append(subsuite("bar")) + + +def test_add_two_tags_filters(): + tag1 = tags("foo") + tag2 = tags("bar") + fl = filterlist([tag1]) + + with pytest.raises(ValueError): + fl.append(tag1) + + fl.append(tag2) + assert len(fl) == 2 + + +def test_filters_run_in_order(): + def a(x, y): + return x + + def b(x, y): + return x + + def c(x, y): + return x + + def d(x, y): + return x + + def e(x, y): + return x + + def f(x, y): + return x + + fl = filterlist([a, b]) + fl.append(c) + fl.extend([d, e]) + fl += [f] + assert [i for i in fl] == [a, b, c, d, e, f] + + +@pytest.fixture(scope="module") +def create_tests(): + def inner(*paths, **defaults): + tests = [] + for path in paths: + if isinstance(path, tuple): + path, kwargs = path + else: + kwargs = {} + + path = mozpath.normpath(path) + manifest = kwargs.pop( + "manifest", + defaults.pop( + "manifest", mozpath.join(mozpath.dirname(path), "manifest.ini") + ), + ) + test = { + "name": mozpath.basename(path), + "path": "/root/" + path, + "relpath": path, + "manifest": "/root/" + manifest, + "manifest_relpath": manifest, + } + test.update(**defaults) + test.update(**kwargs) + tests.append(test) + + # dump tests to stdout for easier debugging on failure + print("The 'create_tests' fixture returned:") + pprint(tests, indent=2) + return tests + + return inner + + +@pytest.fixture +def tests(create_tests): + return create_tests( + "test0", + ("test1", {"skip-if": "foo == 'bar'\nintermittent&&!debug"}), + ("test2", {"run-if": "foo == 'bar'"}), + ("test3", {"fail-if": "foo == 'bar'"}), + ("test4", {"disabled": "some reason"}), + ("test5", {"subsuite": "baz"}), + ("test6", {"subsuite": "baz,foo == 'bar'"}), + ("test7", {"tags": "foo bar"}), + ( + "test8", + {"skip-if": "\nbaz\nfoo == 'bar'\nfoo == 'baz'\nintermittent && debug"}, + ), + ) + + +def test_skip_if(tests): + ref = deepcopy(tests) + tests = list(skip_if(tests, {})) + assert len(tests) == len(ref) + + tests = deepcopy(ref) + tests = list(skip_if(tests, {"foo": "bar"})) + assert "disabled" in tests[1] + assert "disabled" in tests[8] + + +def test_run_if(tests): + ref = deepcopy(tests) + tests = list(run_if(tests, {})) + assert "disabled" in tests[2] + + tests = deepcopy(ref) + tests = list(run_if(tests, {"foo": "bar"})) + assert "disabled" not in tests[2] + + +def test_fail_if(tests): + ref = deepcopy(tests) + tests = list(fail_if(tests, {})) + assert "expected" not in tests[3] + + tests = deepcopy(ref) + tests = list(fail_if(tests, {"foo": "bar"})) + assert tests[3]["expected"] == "fail" + + +def test_enabled(tests): + ref = deepcopy(tests) + tests = list(enabled(tests, {})) + assert ref[4] not in tests + + +def test_subsuite(tests): + sub1 = subsuite() + sub2 = subsuite("baz") + + ref = deepcopy(tests) + tests = list(sub1(tests, {})) + assert ref[5] not in tests + assert len(tests) == len(ref) - 1 + + tests = deepcopy(ref) + tests = list(sub2(tests, {})) + assert len(tests) == 1 + assert ref[5] in tests + + +def test_subsuite_condition(tests): + sub1 = subsuite() + sub2 = subsuite("baz") + + ref = deepcopy(tests) + + tests = list(sub1(tests, {"foo": "bar"})) + assert ref[5] not in tests + assert ref[6] not in tests + + tests = deepcopy(ref) + tests = list(sub2(tests, {"foo": "bar"})) + assert len(tests) == 2 + assert tests[0]["name"] == "test5" + assert tests[1]["name"] == "test6" + + +def test_tags(tests): + ftags1 = tags([]) + ftags2 = tags(["bar", "baz"]) + + ref = deepcopy(tests) + tests = list(ftags1(tests, {})) + assert len(tests) == 0 + + tests = deepcopy(ref) + tests = list(ftags2(tests, {})) + assert len(tests) == 1 + assert ref[7] in tests + + +def test_failures(tests): + ref = deepcopy(tests) + fail1 = failures("intermittent") + tests = list(fail1(tests, {"intermittent": True, "debug": True})) + assert len(tests) == 1 + + tests = deepcopy(ref) + tests = list(fail1(tests, {"intermittent": True})) + assert len(tests) == 1 + + tests = deepcopy(ref) + tests = list(fail1(tests, {})) + assert len(tests) == 0 + + tests = deepcopy(ref) + tests = list(fail1(tests, {"intermittent": False, "debug": True})) + assert len(tests) == 0 + + +def test_pathprefix(create_tests): + tests = create_tests( + "test0", + "subdir/test1", + "subdir/test2", + ("subdir/test3", {"manifest": "manifest.ini"}), + ( + "other/test4", + { + "manifest": "manifest-common.ini", + "ancestor_manifest": "other/manifest.ini", + }, + ), + ) + + def names(items): + return sorted(i["name"] for i in items) + + # relative directory + prefix = pathprefix("subdir") + filtered = prefix(tests, {}) + assert names(filtered) == ["test1", "test2", "test3"] + + # absolute directory + prefix = pathprefix(["/root/subdir"]) + filtered = prefix(tests, {}) + assert names(filtered) == ["test1", "test2", "test3"] + + # relative manifest + prefix = pathprefix(["subdir/manifest.ini"]) + filtered = prefix(tests, {}) + assert names(filtered) == ["test1", "test2"] + + # absolute manifest + prefix = pathprefix(["/root/subdir/manifest.ini"]) + filtered = prefix(tests, {}) + assert names(filtered) == ["test1", "test2"] + + # mixed test and manifest + prefix = pathprefix(["subdir/test2", "manifest.ini"]) + filtered = prefix(tests, {}) + assert names(filtered) == ["test0", "test2", "test3"] + + # relative ancestor manifest + prefix = pathprefix(["other/manifest.ini"]) + filtered = prefix(tests, {}) + assert names(filtered) == ["test4"] + + # absolute ancestor manifest + prefix = pathprefix(["/root/other/manifest.ini"]) + filtered = prefix(tests, {}) + assert names(filtered) == ["test4"] + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/mozbase/manifestparser/tests/test_manifestparser.py b/testing/mozbase/manifestparser/tests/test_manifestparser.py new file mode 100755 index 0000000000..3e6deaceb7 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/test_manifestparser.py @@ -0,0 +1,440 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import shutil +import tempfile +import unittest + +import mozunit +from manifestparser import ManifestParser +from six import StringIO + +here = os.path.dirname(os.path.abspath(__file__)) + + +class TestManifestParser(unittest.TestCase): + """ + Test the manifest parser + + You must have manifestparser installed before running these tests. + Run ``python manifestparser.py setup develop`` with setuptools installed. + """ + + def test_sanity(self): + """Ensure basic parser is sane""" + + parser = ManifestParser() + mozmill_example = os.path.join(here, "mozmill-example.ini") + parser.read(mozmill_example) + tests = parser.tests + self.assertEqual( + len(tests), len(open(mozmill_example).read().strip().splitlines()) + ) + + # Ensure that capitalization and order aren't an issue: + lines = ["[%s]" % test["name"] for test in tests] + self.assertEqual(lines, open(mozmill_example).read().strip().splitlines()) + + # Show how you select subsets of tests: + mozmill_restart_example = os.path.join(here, "mozmill-restart-example.ini") + parser.read(mozmill_restart_example) + restart_tests = parser.get(type="restart") + self.assertTrue(len(restart_tests) < len(parser.tests)) + self.assertEqual( + len(restart_tests), len(parser.get(manifest=mozmill_restart_example)) + ) + self.assertFalse( + [ + test + for test in restart_tests + if test["manifest"] != os.path.join(here, "mozmill-restart-example.ini") + ] + ) + self.assertEqual( + parser.get("name", tags=["foo"]), + [ + "restartTests/testExtensionInstallUninstall/test2.js", + "restartTests/testExtensionInstallUninstall/test1.js", + ], + ) + self.assertEqual( + parser.get("name", foo="bar"), + ["restartTests/testExtensionInstallUninstall/test2.js"], + ) + + def test_include(self): + """Illustrate how include works""" + + include_example = os.path.join(here, "include-example.ini") + parser = ManifestParser(manifests=(include_example,)) + + # All of the tests should be included, in order: + self.assertEqual(parser.get("name"), ["crash-handling", "fleem", "flowers"]) + self.assertEqual( + [ + (test["name"], os.path.basename(test["manifest"])) + for test in parser.tests + ], + [ + ("crash-handling", "bar.ini"), + ("fleem", "include-example.ini"), + ("flowers", "foo.ini"), + ], + ) + + # The including manifest is always reported as a part of the generated test object. + self.assertTrue( + all( + [ + t["ancestor_manifest"] == "include-example.ini" + for t in parser.tests + if t["name"] != "fleem" + ] + ) + ) + + # The manifests should be there too: + self.assertEqual(len(parser.manifests()), 3) + + # We already have the root directory: + self.assertEqual(here, parser.rootdir) + + # DEFAULT values should persist across includes, unless they're + # overwritten. In this example, include-example.ini sets foo=bar, but + # it's overridden to fleem in bar.ini + self.assertEqual(parser.get("name", foo="bar"), ["fleem", "flowers"]) + self.assertEqual(parser.get("name", foo="fleem"), ["crash-handling"]) + + # Passing parameters in the include section allows defining variables in + # the submodule scope: + self.assertEqual(parser.get("name", tags=["red"]), ["flowers"]) + + # However, this should be overridable from the DEFAULT section in the + # included file and that overridable via the key directly connected to + # the test: + self.assertEqual(parser.get(name="flowers")[0]["blue"], "ocean") + self.assertEqual(parser.get(name="flowers")[0]["yellow"], "submarine") + + # You can query multiple times if you need to: + flowers = parser.get(foo="bar") + self.assertEqual(len(flowers), 2) + + # Using the inverse flag should invert the set of tests returned: + self.assertEqual( + parser.get("name", inverse=True, tags=["red"]), ["crash-handling", "fleem"] + ) + + # All of the included tests actually exist: + self.assertEqual([i["name"] for i in parser.missing()], []) + + # Write the output to a manifest: + buffer = StringIO() + parser.write(fp=buffer, global_kwargs={"foo": "bar"}) + expected_output = """[DEFAULT] +foo = bar + +[fleem] + +[include/flowers] +blue = ocean +red = roses +yellow = submarine""" # noqa + + self.assertEqual(buffer.getvalue().strip(), expected_output) + + def test_include_manifest_defaults(self): + """ + Test that manifest_defaults and manifests() are correctly populated + when includes are used. + """ + + include_example = os.path.join(here, "include-example.ini") + noinclude_example = os.path.join(here, "just-defaults.ini") + bar_path = os.path.join(here, "include", "bar.ini") + foo_path = os.path.join(here, "include", "foo.ini") + + parser = ManifestParser( + manifests=(include_example, noinclude_example), rootdir=here + ) + + # Standalone manifests must be appear as-is. + self.assertTrue(include_example in parser.manifest_defaults) + self.assertTrue(noinclude_example in parser.manifest_defaults) + + # Included manifests must only appear together with the parent manifest + # that included the manifest. + self.assertFalse(bar_path in parser.manifest_defaults) + self.assertFalse(foo_path in parser.manifest_defaults) + ancestor_ini = os.path.relpath(include_example, parser.rootdir) + self.assertTrue((ancestor_ini, bar_path) in parser.manifest_defaults) + self.assertTrue((ancestor_ini, foo_path) in parser.manifest_defaults) + + # manifests() must only return file paths (strings). + manifests = parser.manifests() + self.assertEqual(len(manifests), 4) + self.assertIn(foo_path, manifests) + self.assertIn(bar_path, manifests) + self.assertIn(include_example, manifests) + self.assertIn(noinclude_example, manifests) + + def test_include_handle_defaults_False(self): + """ + Test that manifest_defaults and manifests() are correct even when + handle_defaults is set to False. + """ + manifest = os.path.join(here, "include-example.ini") + foo_path = os.path.join(here, "include", "foo.ini") + + parser = ManifestParser( + manifests=(manifest,), handle_defaults=False, rootdir=here + ) + ancestor_ini = os.path.relpath(manifest, parser.rootdir) + + self.assertIn(manifest, parser.manifest_defaults) + self.assertNotIn(foo_path, parser.manifest_defaults) + self.assertIn((ancestor_ini, foo_path), parser.manifest_defaults) + self.assertEqual( + parser.manifest_defaults[manifest], + { + "foo": "bar", + "here": here, + }, + ) + self.assertEqual( + parser.manifest_defaults[(ancestor_ini, foo_path)], + { + "here": os.path.join(here, "include"), + "red": "roses", + "blue": "ocean", + "yellow": "daffodils", + }, + ) + + def test_include_repeated(self): + """ + Test that repeatedly included manifests are independent of each other. + """ + include_example = os.path.join(here, "include-example.ini") + included_foo = os.path.join(here, "include", "foo.ini") + + # In the expected output, blue and yellow have the values from foo.ini + # (ocean, submarine) instead of the ones from include-example.ini + # (violets, daffodils), because the defaults in the included file take + # precedence over the values from the parent. + include_output = """[include/crash-handling] +foo = fleem + +[fleem] +foo = bar + +[include/flowers] +blue = ocean +foo = bar +red = roses +yellow = submarine + +""" + included_output = """[include/flowers] +blue = ocean +yellow = submarine + +""" + + parser = ManifestParser(manifests=(include_example, included_foo), rootdir=here) + self.assertEqual( + parser.get("name"), ["crash-handling", "fleem", "flowers", "flowers"] + ) + self.assertEqual( + [ + (test["name"], os.path.basename(test["manifest"])) + for test in parser.tests + ], + [ + ("crash-handling", "bar.ini"), + ("fleem", "include-example.ini"), + ("flowers", "foo.ini"), + ("flowers", "foo.ini"), + ], + ) + self.check_included_repeat( + parser, + parser.tests[3], + parser.tests[2], + "%s%s" % (include_output, included_output), + ) + + # Same tests, but with the load order of the manifests swapped. + parser = ManifestParser(manifests=(included_foo, include_example), rootdir=here) + self.assertEqual( + parser.get("name"), ["flowers", "crash-handling", "fleem", "flowers"] + ) + self.assertEqual( + [ + (test["name"], os.path.basename(test["manifest"])) + for test in parser.tests + ], + [ + ("flowers", "foo.ini"), + ("crash-handling", "bar.ini"), + ("fleem", "include-example.ini"), + ("flowers", "foo.ini"), + ], + ) + self.check_included_repeat( + parser, + parser.tests[0], + parser.tests[3], + "%s%s" % (included_output, include_output), + ) + + def check_included_repeat( + self, parser, isolated_test, included_test, expected_output + ): + include_example = os.path.join(here, "include-example.ini") + included_foo = os.path.join(here, "include", "foo.ini") + ancestor_ini = os.path.relpath(include_example, parser.rootdir) + manifest_default_key = (ancestor_ini, included_foo) + + self.assertFalse("ancestor_manifest" in isolated_test) + self.assertEqual(included_test["ancestor_manifest"], "include-example.ini") + + self.assertTrue(include_example in parser.manifest_defaults) + self.assertTrue(included_foo in parser.manifest_defaults) + self.assertTrue(manifest_default_key in parser.manifest_defaults) + self.assertEqual( + parser.manifest_defaults[manifest_default_key], + { + "foo": "bar", + "here": os.path.join(here, "include"), + "red": "roses", + "blue": "ocean", + "yellow": "daffodils", + }, + ) + + buffer = StringIO() + parser.write(fp=buffer) + self.assertEqual(buffer.getvalue(), expected_output) + + def test_invalid_path(self): + """ + Test invalid path should not throw when not strict + """ + manifest = os.path.join(here, "include-invalid.ini") + ManifestParser(manifests=(manifest,), strict=False) + + def test_copy(self): + """Test our ability to copy a set of manifests""" + + tempdir = tempfile.mkdtemp() + include_example = os.path.join(here, "include-example.ini") + manifest = ManifestParser(manifests=(include_example,)) + manifest.copy(tempdir) + self.assertEqual( + sorted(os.listdir(tempdir)), ["fleem", "include", "include-example.ini"] + ) + self.assertEqual( + sorted(os.listdir(os.path.join(tempdir, "include"))), + ["bar.ini", "crash-handling", "flowers", "foo.ini"], + ) + from_manifest = ManifestParser(manifests=(include_example,)) + to_manifest = os.path.join(tempdir, "include-example.ini") + to_manifest = ManifestParser(manifests=(to_manifest,)) + self.assertEqual(to_manifest.get("name"), from_manifest.get("name")) + shutil.rmtree(tempdir) + + def test_path_override(self): + """You can override the path in the section too. + This shows that you can use a relative path""" + path_example = os.path.join(here, "path-example.ini") + manifest = ManifestParser(manifests=(path_example,)) + self.assertEqual(manifest.tests[0]["path"], os.path.join(here, "fleem")) + + def test_relative_path(self): + """ + Relative test paths are correctly calculated. + """ + relative_path = os.path.join(here, "relative-path.ini") + manifest = ManifestParser(manifests=(relative_path,)) + self.assertEqual( + manifest.tests[0]["path"], os.path.join(os.path.dirname(here), "fleem") + ) + self.assertEqual(manifest.tests[0]["relpath"], os.path.join("..", "fleem")) + self.assertEqual( + manifest.tests[1]["relpath"], os.path.join("..", "testsSIBLING", "example") + ) + + def test_path_from_fd(self): + """ + Test paths are left untouched when manifest is a file-like object. + """ + fp = StringIO("[section]\npath=fleem") + manifest = ManifestParser(manifests=(fp,)) + self.assertEqual(manifest.tests[0]["path"], "fleem") + self.assertEqual(manifest.tests[0]["relpath"], "fleem") + self.assertEqual(manifest.tests[0]["manifest"], None) + + def test_comments(self): + """ + ensure comments work, see + https://bugzilla.mozilla.org/show_bug.cgi?id=813674 + """ + comment_example = os.path.join(here, "comment-example.ini") + manifest = ManifestParser(manifests=(comment_example,)) + self.assertEqual(len(manifest.tests), 8) + names = [i["name"] for i in manifest.tests] + self.assertFalse("test_0202_app_launch_apply_update_dirlocked.js" in names) + + def test_verifyDirectory(self): + + directory = os.path.join(here, "verifyDirectory") + + # correct manifest + manifest_path = os.path.join(directory, "verifyDirectory.ini") + manifest = ManifestParser(manifests=(manifest_path,)) + missing = manifest.verifyDirectory(directory, extensions=(".js",)) + self.assertEqual(missing, (set(), set())) + + # manifest is missing test_1.js + test_1 = os.path.join(directory, "test_1.js") + manifest_path = os.path.join(directory, "verifyDirectory_incomplete.ini") + manifest = ManifestParser(manifests=(manifest_path,)) + missing = manifest.verifyDirectory(directory, extensions=(".js",)) + self.assertEqual(missing, (set(), set([test_1]))) + + # filesystem is missing test_notappearinginthisfilm.js + missing_test = os.path.join(directory, "test_notappearinginthisfilm.js") + manifest_path = os.path.join(directory, "verifyDirectory_toocomplete.ini") + manifest = ManifestParser(manifests=(manifest_path,)) + missing = manifest.verifyDirectory(directory, extensions=(".js",)) + self.assertEqual(missing, (set([missing_test]), set())) + + def test_just_defaults(self): + """Ensure a manifest with just a DEFAULT section exposes that data.""" + + parser = ManifestParser() + manifest = os.path.join(here, "just-defaults.ini") + parser.read(manifest) + self.assertEqual(len(parser.tests), 0) + self.assertTrue(manifest in parser.manifest_defaults) + self.assertEqual(parser.manifest_defaults[manifest]["foo"], "bar") + + def test_manifest_list(self): + """ + Ensure a manifest with just a DEFAULT section still returns + itself from the manifests() method. + """ + + parser = ManifestParser() + manifest = os.path.join(here, "no-tests.ini") + parser.read(manifest) + self.assertEqual(len(parser.tests), 0) + self.assertTrue(len(parser.manifests()) == 1) + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/mozbase/manifestparser/tests/test_read_ini.py b/testing/mozbase/manifestparser/tests/test_read_ini.py new file mode 100755 index 0000000000..257054ee52 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/test_read_ini.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python + +""" +test .ini parsing + +ensure our .ini parser is doing what we want; to be deprecated for +python's standard ConfigParser when 2.7 is reality so OrderedDict +is the default: + +http://docs.python.org/2/library/configparser.html +""" + +from textwrap import dedent + +import mozunit +import pytest +from manifestparser import read_ini +from six import StringIO + + +@pytest.fixture(scope="module") +def parse_manifest(): + def inner(string, **kwargs): + buf = StringIO() + buf.write(dedent(string)) + buf.seek(0) + return read_ini(buf, **kwargs)[0] + + return inner + + +def test_inline_comments(parse_manifest): + result = parse_manifest( + """ + [test_felinicity.py] + kittens = true # This test requires kittens + cats = false#but not cats + """ + )[0][1] + + # make sure inline comments get stripped out, but comments without a space in front don't + assert result["kittens"] == "true" + assert result["cats"] == "false#but not cats" + + +def test_line_continuation(parse_manifest): + result = parse_manifest( + """ + [test_caninicity.py] + breeds = + sheppard + retriever + terrier + + [test_cats_and_dogs.py] + cats=yep + dogs= + yep + yep + birds=nope + fish=nope + """ + ) + assert result[0][1]["breeds"].split() == ["sheppard", "retriever", "terrier"] + assert result[1][1]["cats"] == "yep" + assert result[1][1]["dogs"].split() == ["yep", "yep"] + assert result[1][1]["birds"].split() == ["nope", "fish=nope"] + + +def test_dupes_error(parse_manifest): + dupes = """ + [test_dupes.py] + foo = bar + foo = baz + """ + with pytest.raises(AssertionError): + parse_manifest(dupes, strict=True) + + with pytest.raises(AssertionError): + parse_manifest(dupes, strict=False) + + +def test_defaults_handling(parse_manifest): + manifest = """ + [DEFAULT] + flower = rose + skip-if = true + + [test_defaults] + """ + + result = parse_manifest(manifest)[0][1] + assert result["flower"] == "rose" + assert result["skip-if"] == "true" + + result = parse_manifest( + manifest, + defaults={ + "flower": "tulip", + "colour": "pink", + "skip-if": "false", + }, + )[0][1] + assert result["flower"] == "rose" + assert result["colour"] == "pink" + assert result["skip-if"] == "false\ntrue" + + result = parse_manifest(manifest.replace("DEFAULT", "default"))[0][1] + assert result["flower"] == "rose" + assert result["skip-if"] == "true" + + +def test_multiline_skip(parse_manifest): + manifest = """ + [test_multiline_skip] + skip-if = + os == "mac" # bug 123 + os == "linux" && debug # bug 456 + """ + + result = parse_manifest(manifest)[0][1] + assert ( + result["skip-if"].replace("\r\n", "\n") + == dedent( + """ + os == "mac" + os == "linux" && debug + """ + ).rstrip() + ) + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/mozbase/manifestparser/tests/test_testmanifest.py b/testing/mozbase/manifestparser/tests/test_testmanifest.py new file mode 100644 index 0000000000..0aef82b00f --- /dev/null +++ b/testing/mozbase/manifestparser/tests/test_testmanifest.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python + +import os +import shutil +import tempfile +import unittest + +import mozunit +from manifestparser import ParseError, TestManifest +from manifestparser.filters import subsuite + +here = os.path.dirname(os.path.abspath(__file__)) + + +class TestTestManifest(unittest.TestCase): + """Test the Test Manifest""" + + def test_testmanifest(self): + # Test filtering based on platform: + filter_example = os.path.join(here, "filter-example.ini") + manifest = TestManifest(manifests=(filter_example,), strict=False) + self.assertEqual( + [ + i["name"] + for i in manifest.active_tests(os="win", disabled=False, exists=False) + ], + ["windowstest", "fleem"], + ) + self.assertEqual( + [ + i["name"] + for i in manifest.active_tests(os="linux", disabled=False, exists=False) + ], + ["fleem", "linuxtest"], + ) + + # Look for existing tests. There is only one: + self.assertEqual([i["name"] for i in manifest.active_tests()], ["fleem"]) + + # You should be able to expect failures: + last = manifest.active_tests(exists=False, toolkit="gtk")[-1] + self.assertEqual(last["name"], "linuxtest") + self.assertEqual(last["expected"], "pass") + last = manifest.active_tests(exists=False, toolkit="cocoa")[-1] + self.assertEqual(last["expected"], "fail") + + def test_missing_paths(self): + """ + Test paths that don't exist raise an exception in strict mode. + """ + tempdir = tempfile.mkdtemp() + + missing_path = os.path.join(here, "missing-path.ini") + manifest = TestManifest(manifests=(missing_path,), strict=True) + self.assertRaises(IOError, manifest.active_tests) + self.assertRaises(IOError, manifest.copy, tempdir) + self.assertRaises(IOError, manifest.update, tempdir) + + shutil.rmtree(tempdir) + + def test_comments(self): + """ + ensure comments work, see + https://bugzilla.mozilla.org/show_bug.cgi?id=813674 + """ + comment_example = os.path.join(here, "comment-example.ini") + manifest = TestManifest(manifests=(comment_example,)) + self.assertEqual(len(manifest.tests), 8) + names = [i["name"] for i in manifest.tests] + self.assertFalse("test_0202_app_launch_apply_update_dirlocked.js" in names) + + def test_manifest_subsuites(self): + """ + test subsuites and conditional subsuites + """ + relative_path = os.path.join(here, "subsuite.ini") + manifest = TestManifest(manifests=(relative_path,)) + info = {"foo": "bar"} + + # 6 tests total + tests = manifest.active_tests(exists=False, **info) + self.assertEqual(len(tests), 6) + + # only 3 tests for subsuite bar when foo==bar + tests = manifest.active_tests(exists=False, filters=[subsuite("bar")], **info) + self.assertEqual(len(tests), 3) + + # only 1 test for subsuite baz, regardless of conditions + other = {"something": "else"} + tests = manifest.active_tests(exists=False, filters=[subsuite("baz")], **info) + self.assertEqual(len(tests), 1) + tests = manifest.active_tests(exists=False, filters=[subsuite("baz")], **other) + self.assertEqual(len(tests), 1) + + # 4 tests match when the condition doesn't match (all tests except + # the unconditional subsuite) + info = {"foo": "blah"} + tests = manifest.active_tests(exists=False, filters=[subsuite()], **info) + self.assertEqual(len(tests), 5) + + # test for illegal subsuite value + manifest.tests[0]["subsuite"] = 'subsuite=bar,foo=="bar",type="nothing"' + with self.assertRaises(ParseError): + manifest.active_tests(exists=False, filters=[subsuite("foo")], **info) + + def test_none_and_empty_manifest(self): + """ + Test TestManifest for None and empty manifest, see + https://bugzilla.mozilla.org/show_bug.cgi?id=1087682 + """ + none_manifest = TestManifest(manifests=None, strict=False) + self.assertEqual(len(none_manifest.test_paths()), 0) + self.assertEqual(len(none_manifest.active_tests()), 0) + + empty_manifest = TestManifest(manifests=[], strict=False) + self.assertEqual(len(empty_manifest.test_paths()), 0) + self.assertEqual(len(empty_manifest.active_tests()), 0) + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/mozbase/manifestparser/tests/test_util.py b/testing/mozbase/manifestparser/tests/test_util.py new file mode 100644 index 0000000000..e32ecbab79 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/test_util.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python + +""" +Test how our utility functions are working. +""" + +from textwrap import dedent + +import mozunit +import pytest +from manifestparser import read_ini +from manifestparser.util import evaluate_list_from_string +from six import StringIO + + +@pytest.fixture(scope="module") +def parse_manifest(): + def inner(string, **kwargs): + buf = StringIO() + buf.write(dedent(string)) + buf.seek(0) + return read_ini(buf, **kwargs)[0] + + return inner + + +@pytest.mark.parametrize( + "test_manifest, expected_list", + [ + [ + """ + [test_felinicity.py] + kittens = true + cats = + "I", + "Am", + "A", + "Cat", + """, + ["I", "Am", "A", "Cat"], + ], + [ + """ + [test_felinicity.py] + kittens = true + cats = + ["I", 1], + ["Am", 2], + ["A", 3], + ["Cat", 4], + """, + [ + ["I", 1], + ["Am", 2], + ["A", 3], + ["Cat", 4], + ], + ], + ], +) +def test_string_to_list_conversion(test_manifest, expected_list, parse_manifest): + parsed_tests = parse_manifest(test_manifest) + assert evaluate_list_from_string(parsed_tests[0][1]["cats"]) == expected_list + + +@pytest.mark.parametrize( + "test_manifest, failure", + [ + [ + """ + # This will fail since the elements are not enlosed in quotes + [test_felinicity.py] + kittens = true + cats = + I, + Am, + A, + Cat, + """, + ValueError, + ], + [ + """ + # This will fail since the syntax is incorrect + [test_felinicity.py] + kittens = true + cats = + ["I", 1, + ["Am", 2, + ["A", 3], + ["Cat", 4], + """, + SyntaxError, + ], + ], +) +def test_string_to_list_conversion_failures(test_manifest, failure, parse_manifest): + parsed_tests = parse_manifest(test_manifest) + with pytest.raises(failure): + evaluate_list_from_string(parsed_tests[0][1]["cats"]) + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/mozbase/manifestparser/tests/verifyDirectory/subdir/manifest.ini b/testing/mozbase/manifestparser/tests/verifyDirectory/subdir/manifest.ini new file mode 100644 index 0000000000..509ebd62ef --- /dev/null +++ b/testing/mozbase/manifestparser/tests/verifyDirectory/subdir/manifest.ini @@ -0,0 +1 @@ +[test_sub.js] diff --git a/testing/mozbase/manifestparser/tests/verifyDirectory/subdir/test_sub.js b/testing/mozbase/manifestparser/tests/verifyDirectory/subdir/test_sub.js new file mode 100644 index 0000000000..df48720d9d --- /dev/null +++ b/testing/mozbase/manifestparser/tests/verifyDirectory/subdir/test_sub.js @@ -0,0 +1 @@ +// test_sub.js diff --git a/testing/mozbase/manifestparser/tests/verifyDirectory/test_1.js b/testing/mozbase/manifestparser/tests/verifyDirectory/test_1.js new file mode 100644 index 0000000000..c5a966f46a --- /dev/null +++ b/testing/mozbase/manifestparser/tests/verifyDirectory/test_1.js @@ -0,0 +1 @@ +// test_1.js diff --git a/testing/mozbase/manifestparser/tests/verifyDirectory/test_2.js b/testing/mozbase/manifestparser/tests/verifyDirectory/test_2.js new file mode 100644 index 0000000000..d8648599c5 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/verifyDirectory/test_2.js @@ -0,0 +1 @@ +// test_2.js diff --git a/testing/mozbase/manifestparser/tests/verifyDirectory/test_3.js b/testing/mozbase/manifestparser/tests/verifyDirectory/test_3.js new file mode 100644 index 0000000000..794bc2c341 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/verifyDirectory/test_3.js @@ -0,0 +1 @@ +// test_3.js diff --git a/testing/mozbase/manifestparser/tests/verifyDirectory/verifyDirectory.ini b/testing/mozbase/manifestparser/tests/verifyDirectory/verifyDirectory.ini new file mode 100644 index 0000000000..10e0c79c81 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/verifyDirectory/verifyDirectory.ini @@ -0,0 +1,4 @@ +[test_1.js] +[test_2.js] +[test_3.js] +[include:subdir/manifest.ini] diff --git a/testing/mozbase/manifestparser/tests/verifyDirectory/verifyDirectory_incomplete.ini b/testing/mozbase/manifestparser/tests/verifyDirectory/verifyDirectory_incomplete.ini new file mode 100644 index 0000000000..cde526acfc --- /dev/null +++ b/testing/mozbase/manifestparser/tests/verifyDirectory/verifyDirectory_incomplete.ini @@ -0,0 +1,3 @@ +[test_2.js] +[test_3.js] +[include:subdir/manifest.ini] diff --git a/testing/mozbase/manifestparser/tests/verifyDirectory/verifyDirectory_toocomplete.ini b/testing/mozbase/manifestparser/tests/verifyDirectory/verifyDirectory_toocomplete.ini new file mode 100644 index 0000000000..88994ae26f --- /dev/null +++ b/testing/mozbase/manifestparser/tests/verifyDirectory/verifyDirectory_toocomplete.ini @@ -0,0 +1,5 @@ +[test_1.js] +[test_2.js] +[test_3.js] +[test_notappearinginthisfilm.js] +[include:subdir/manifest.ini] |