diff options
Diffstat (limited to 'testing/test/test_skipfails.py')
| -rw-r--r-- | testing/test/test_skipfails.py | 277 |
1 files changed, 255 insertions, 22 deletions
diff --git a/testing/test/test_skipfails.py b/testing/test/test_skipfails.py index 9dce0a9e23..cc6af850aa 100644 --- a/testing/test/test_skipfails.py +++ b/testing/test/test_skipfails.py @@ -8,13 +8,7 @@ from pathlib import Path import pytest from mozunit import main -from skipfails import ( - MOCK_BUG_DEFAULTS, - MOCK_TASK_DEFAULTS, - MOCK_TASK_INITS, - Mock, - Skipfails, -) +from skipfails import Skipfails DATA_PATH = Path(__file__).with_name("data") @@ -63,11 +57,9 @@ def get_failures(tasks_name, exp_f_name, task_details=None): if task_details is not None: # preload task details cache, if needed for task_id in task_details: sf.tasks[task_id] = task_details[task_id] - tasks_fp = DATA_PATH.joinpath(tasks_name).open("r", encoding="utf-8") - tasks = json.load(tasks_fp) - tasks = [Mock(task, MOCK_TASK_DEFAULTS, MOCK_TASK_INITS) for task in tasks] - exp_f_fp = DATA_PATH.joinpath(exp_f_name).open("r", encoding="utf-8") - expected_failures = exp_f_fp.read().strip() + tasks = sf.read_tasks(DATA_PATH.joinpath(tasks_name)) + exp_f = sf.read_failures(DATA_PATH.joinpath(exp_f_name)) + expected_failures = json.dumps(exp_f, indent=2, sort_keys=True).strip() failures = sf.get_failures(tasks) actual_failures = json.dumps(failures, indent=2, sort_keys=True).strip() assert actual_failures == expected_failures @@ -132,6 +124,31 @@ def test_get_failures_5(): get_failures("wayland-tasks-5.json", "wayland-failures-5.json", task_details) +def test_get_failures_6(): + """Test get_failures 6""" + + task_details = { + "AKYqxtoWStigj_5yHVqAeg": { + "expires": "2024-03-19T03:29:11.050Z", + "extra": { + "suite": "web-platform-tests", + "test-setting": { + "build": { + "type": "opt", + "shippable": True, + }, + "platform": { + "arch": "32", + "os": {"name": "linux", "version": "1804"}, + }, + "runtime": {}, + }, + }, + } + } + get_failures("wpt-tasks-1.json", "wpt-failures-1.json", task_details) + + def test_get_bug_by_id(): """Test get_bug_by_id""" @@ -139,12 +156,10 @@ def test_get_bug_by_id(): id = 1682371 # preload bug cache bugs_filename = f"bug-{id}.json" - bugs_fp = DATA_PATH.joinpath(bugs_filename).open("r", encoding="utf-8") - bugs_data = json.load(bugs_fp) - preload_bugs = [Mock(b, MOCK_BUG_DEFAULTS) for b in bugs_data] - sf.bugs = preload_bugs + sf.bugs = sf.read_bugs(DATA_PATH.joinpath(bugs_filename)) # function under test bug = sf.get_bug_by_id(id) + assert bug is not None assert bug.id == id assert bug.product == "Testing" assert bug.component == "General" @@ -161,10 +176,7 @@ def test_get_bugs_by_summary(): id = 1682371 # preload bug cache bugs_filename = f"bug-{id}.json" - bugs_fp = DATA_PATH.joinpath(bugs_filename).open("r", encoding="utf-8") - bugs_data = json.load(bugs_fp) - preload_bugs = [Mock(b, MOCK_BUG_DEFAULTS) for b in bugs_data] - sf.bugs = preload_bugs + sf.bugs = sf.read_bugs(DATA_PATH.joinpath(bugs_filename)) # function under test summary = "create tool to quickly parse and identify all failures from a try push and ideally annotate manifests" bugs = sf.get_bugs_by_summary(summary) @@ -191,6 +203,7 @@ def test_get_variants(): def test_task_to_skip_if(): """Test task_to_skip_if""" + sf = Skipfails() # preload task cache task_id = "UP-t3xrGSDWvUNjFGIt_aQ" task_details = { @@ -207,11 +220,231 @@ def test_task_to_skip_if(): }, }, } - sf = Skipfails() sf.tasks[task_id] = task_details # function under test skip_if = sf.task_to_skip_if(task_id) - assert skip_if == "os == 'win' && os_version == '11' && bits == '32' && debug" + assert skip_if == "win11_2009 && processor == 'x86' && debug" + task_id = "I3iXyGDATDSDyzGh4YfNJw" + task_details = { + "expires": "2024-01-09T16:05:56.825Z", + "extra": { + "suite": "web-platform-tests-crashtest", + "test-setting": { + "build": {"type": "debug"}, + "platform": { + "arch": "64", + "os": {"name": "macosx", "version": "1015"}, + }, + "runtime": {"webrender-sw": True}, + }, + }, + } + sf.tasks[task_id] = task_details + # function under test + skip_if = sf.task_to_skip_if(task_id) + assert ( + skip_if + == "os == 'mac' && os_version == '10.15' && processor == 'x86_64' && debug && swgl" + ) + task_id = "bAkMaQIVQp6oeEIW6fzBDw" + task_details = { + "expires": "2024-01-09T16:05:56.825Z", + "extra": { + "suite": "mochitest-media", + "test-setting": { + "build": {"type": "debug"}, + "platform": { + "arch": "aarch64", + "os": {"name": "macosx", "version": "1100"}, + }, + "runtime": {"webrender-sw": True}, + }, + }, + } + sf.tasks[task_id] = task_details + # function under test + skip_if = sf.task_to_skip_if(task_id) + assert ( + skip_if + == "os == 'mac' && os_version == '11.00' && processor == 'aarch64' && debug && swgl" + ) + + +def test_task_to_skip_if_wpt(): + """Test task_to_skip_if_wpt""" + + # preload task cache + task_id = "AKYqxtoWStigj_5yHVqAeg" + task_details = { + "expires": "2024-03-19T03:29:11.050Z", + "extra": { + "suite": "web-platform-tests", + "test-setting": { + "build": { + "type": "opt", + "shippable": True, + }, + "platform": { + "arch": "32", + "os": {"name": "linux", "version": "1804"}, + }, + "runtime": {}, + }, + }, + } + sf = Skipfails() + sf.tasks[task_id] = task_details + # function under test + skip_if = sf.task_to_skip_if(task_id, True) + assert ( + skip_if + == 'os == "linux" and os_version == "18.04" and processor == "x86" and not debug' + ) + + +def test_wpt_add_skip_if(): + """Test wpt_add_skip_if""" + + sf = Skipfails() + manifest_before1 = "" + anyjs = {} + filename = "myfile.html" + anyjs[filename] = False + skip_if = 'os == "linux" and processor == "x86" and not debug' + bug_reference = "Bug 123" + disabled = " disabled:\n" + condition = " if " + skip_if + ": " + bug_reference + "\n" + manifest_str = sf.wpt_add_skip_if(manifest_before1, anyjs, skip_if, bug_reference) + manifest_expected1 = "[myfile.html]\n" + disabled + condition + "\n" + assert manifest_str == manifest_expected1 + manifest_before2 = """[myfile.html] + expected: + if fission: [OK, FAIL] + [< [Test 5\\] 1 out of 2 assertions were failed.] + expected: FAIL +""" + manifest_expected2 = ( + """[myfile.html] + expected: + if fission: [OK, FAIL] +""" + + disabled + + condition + + """ + [< [Test 5\\] 1 out of 2 assertions were failed.] + expected: FAIL +""" + ) + anyjs[filename] = False + manifest_str = sf.wpt_add_skip_if(manifest_before2, anyjs, skip_if, bug_reference) + assert manifest_str == manifest_expected2 + anyjs[filename] = False + manifest_str = sf.wpt_add_skip_if(manifest_expected2, anyjs, skip_if, bug_reference) + assert manifest_str == manifest_expected2 + manifest_before4 = """;https: //bugzilla.mozilla.org/show_bug.cgi?id=1838684 +expected: [FAIL, PASS] +[custom-highlight-painting-overlapping-highlights-002.html] + expected: + [PASS, FAIL] +""" + anyjs[filename] = False + manifest_str = sf.wpt_add_skip_if(manifest_before4, anyjs, skip_if, bug_reference) + manifest_expected4 = manifest_before4 + "\n" + manifest_expected1 + assert manifest_str == manifest_expected4 + manifest_before5 = """[myfile.html] + disabled: + if win11_2009 && processor == '32' && debug: Bug 456 +""" + anyjs[filename] = False + manifest_str = sf.wpt_add_skip_if(manifest_before5, anyjs, skip_if, bug_reference) + manifest_expected5 = manifest_before5 + condition + "\n" + assert manifest_str == manifest_expected5 + manifest_before6 = """[myfile.html] + [Window Size] + expected: + if product == "firefox_android": FAIL +""" + anyjs[filename] = False + manifest_str = sf.wpt_add_skip_if(manifest_before6, anyjs, skip_if, bug_reference) + manifest_expected6 = ( + """[myfile.html] + disabled: +""" + + condition + + """ + [Window Size] + expected: + if product == "firefox_android": FAIL +""" + ) + assert manifest_str == manifest_expected6 + manifest_before7 = """[myfile.html] + fuzzy: + if os == "android": maxDifference=0-255;totalPixels=0-105 # bug 1392254 + if os == "linux": maxDifference=0-255;totalPixels=0-136 # bug 1599638 + maxDifference=0-1;totalPixels=0-80 + disabled: + if os == "win": https://bugzilla.mozilla.org/show_bug.cgi?id=1314684 +""" + anyjs[filename] = False + manifest_str = sf.wpt_add_skip_if(manifest_before7, anyjs, skip_if, bug_reference) + manifest_expected7 = manifest_before7 + condition + "\n" + assert manifest_str == manifest_expected7 + manifest_before8 = """[myfile.html] + disabled: + if os == "linux" and os_version == "22.04" and processor == "x86_64" and debug and display == "wayland": Bug TBD + expected: + if swgl and (os == "linux") and debug and not fission: [PASS, FAIL] + +""" + anyjs[filename] = False + manifest_str = sf.wpt_add_skip_if(manifest_before8, anyjs, skip_if, bug_reference) + manifest_expected8 = ( + """[myfile.html] + disabled: + if os == "linux" and os_version == "22.04" and processor == "x86_64" and debug and display == "wayland": Bug TBD +""" + + condition + + """ expected: + if swgl and (os == "linux") and debug and not fission: [PASS, FAIL] + +""" + ) + assert manifest_str == manifest_expected8 + manifest_before9 = """[myfile.html] + disabled: + if os == "linux" and os_version == "22.04" and processor == "x86_64" and debug and display == "wayland": Bug TBD + +""" + anyjs[filename] = False + manifest_str = sf.wpt_add_skip_if(manifest_before9, anyjs, skip_if, bug_reference) + manifest_expected9 = ( + """[myfile.html] + disabled: + if os == "linux" and os_version == "22.04" and processor == "x86_64" and debug and display == "wayland": Bug TBD +""" + + condition + + "\n" + ) + assert manifest_str == manifest_expected9 + manifest_before10 = """[myfile.html] + disabled: + if os == "linux" and os_version == "22.04" and processor == "x86_64" and not debug and display == "wayland": Bug TBD + + [3P fetch: Cross site window setting HTTP cookies] +""" + anyjs[filename] = False + manifest_str = sf.wpt_add_skip_if(manifest_before10, anyjs, skip_if, bug_reference) + manifest_expected10 = ( + """[myfile.html] + disabled: + if os == "linux" and os_version == "22.04" and processor == "x86_64" and not debug and display == "wayland": Bug TBD +""" + + condition + + """\n [3P fetch: Cross site window setting HTTP cookies] +""" + ) + assert manifest_str == manifest_expected10 def test_get_filename_in_manifest(): |