1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
|
#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# jit_test.py -- Python harness for JavaScript trace tests.
import os
import re
import sys
import traceback
from collections import namedtuple
if sys.platform.startswith("linux") or sys.platform.startswith("darwin"):
from .tasks_unix import run_all_tests
else:
from .tasks_win import run_all_tests
from .progressbar import NullProgressBar, ProgressBar
from .results import escape_cmdline
from .structuredlog import TestLogger
from .tempfile import TemporaryDirectory
TESTS_LIB_DIR = os.path.dirname(os.path.abspath(__file__))
JS_DIR = os.path.dirname(os.path.dirname(TESTS_LIB_DIR))
TOP_SRC_DIR = os.path.dirname(os.path.dirname(JS_DIR))
TEST_DIR = os.path.join(JS_DIR, "jit-test", "tests")
LIB_DIR = os.path.join(JS_DIR, "jit-test", "lib") + os.path.sep
MODULE_DIR = os.path.join(JS_DIR, "jit-test", "modules") + os.path.sep
SHELL_XDR = "shell.xdr"
# Backported from Python 3.1 posixpath.py
def _relpath(path, start=None):
"""Return a relative version of a path"""
if not path:
raise ValueError("no path specified")
if start is None:
start = os.curdir
start_list = os.path.abspath(start).split(os.sep)
path_list = os.path.abspath(path).split(os.sep)
# Work out how much of the filepath is shared by start and path.
i = len(os.path.commonprefix([start_list, path_list]))
rel_list = [os.pardir] * (len(start_list) - i) + path_list[i:]
if not rel_list:
return os.curdir
return os.path.join(*rel_list)
# Mapping of Python chars to their javascript string representation.
QUOTE_MAP = {
"\\": "\\\\",
"\b": "\\b",
"\f": "\\f",
"\n": "\\n",
"\r": "\\r",
"\t": "\\t",
"\v": "\\v",
}
# Quote the string S, javascript style.
def js_quote(quote, s):
result = quote
for c in s:
if c == quote:
result += "\\" + quote
elif c in QUOTE_MAP:
result += QUOTE_MAP[c]
else:
result += c
result += quote
return result
os.path.relpath = _relpath
def extend_condition(condition, value):
if condition:
condition += " || "
condition += "({})".format(value)
return condition
class JitTest:
VALGRIND_CMD = []
paths = (d for d in os.environ["PATH"].split(os.pathsep))
valgrinds = (os.path.join(d, "valgrind") for d in paths)
if any(os.path.exists(p) for p in valgrinds):
VALGRIND_CMD = [
"valgrind",
"-q",
"--smc-check=all-non-file",
"--error-exitcode=1",
"--gen-suppressions=all",
"--show-possibly-lost=no",
"--leak-check=full",
]
if os.uname()[0] == "Darwin":
VALGRIND_CMD.append("--dsymutil=yes")
del paths
del valgrinds
def __init__(self, path):
# Absolute path of the test file.
self.path = path
# Path relative to the top mozilla/ directory.
self.relpath_top = os.path.relpath(path, TOP_SRC_DIR)
# Path relative to mozilla/js/src/jit-test/tests/.
self.relpath_tests = os.path.relpath(path, TEST_DIR)
# jit flags to enable
self.jitflags = []
# True means the test is slow-running
self.slow = False
# True means that OOM is not considered a failure
self.allow_oom = False
# True means CrashAtUnhandlableOOM is not considered a failure
self.allow_unhandlable_oom = False
# True means that hitting recursion the limits is not considered a failure.
self.allow_overrecursed = False
# True means run under valgrind
self.valgrind = False
# True means force Pacific time for the test
self.tz_pacific = False
# Additional files to include, in addition to prologue.js
self.other_lib_includes = []
self.other_script_includes = []
# List of other configurations to test with.
self.test_also = []
# List of other configurations to test with all existing variants.
self.test_join = []
# Errors to expect and consider passing
self.expect_error = ""
# Exit status to expect from shell
self.expect_status = 0
# Exit status or error output.
self.expect_crash = False
self.is_module = False
# Reflect.stringify implementation to test
self.test_reflect_stringify = None
# Use self-hosted XDR instead of parsing the source stored in the binary.
self.selfhosted_xdr_path = None
self.selfhosted_xdr_mode = "off"
# Skip-if condition. We don't have a xulrunner, but we can ask the shell
# directly.
self.skip_if_cond = ""
self.skip_variant_if_cond = {}
# Expected by the test runner. Always true for jit-tests.
self.enable = True
def copy(self):
t = JitTest(self.path)
t.jitflags = self.jitflags[:]
t.slow = self.slow
t.allow_oom = self.allow_oom
t.allow_unhandlable_oom = self.allow_unhandlable_oom
t.allow_overrecursed = self.allow_overrecursed
t.valgrind = self.valgrind
t.tz_pacific = self.tz_pacific
t.other_lib_includes = self.other_lib_includes[:]
t.other_script_includes = self.other_script_includes[:]
t.test_also = self.test_also
t.test_join = self.test_join
t.expect_error = self.expect_error
t.expect_status = self.expect_status
t.expect_crash = self.expect_crash
t.test_reflect_stringify = self.test_reflect_stringify
t.selfhosted_xdr_path = self.selfhosted_xdr_path
t.selfhosted_xdr_mode = self.selfhosted_xdr_mode
t.enable = True
t.is_module = self.is_module
t.skip_if_cond = self.skip_if_cond
t.skip_variant_if_cond = self.skip_variant_if_cond
return t
def copy_and_extend_jitflags(self, variant):
t = self.copy()
t.jitflags.extend(variant)
for flags in variant:
if flags in self.skip_variant_if_cond:
t.skip_if_cond = extend_condition(
t.skip_if_cond, self.skip_variant_if_cond[flags]
)
return t
def copy_variants(self, variants):
# Append variants to be tested in addition to the current set of tests.
variants = variants + self.test_also
# For each existing variant, duplicates it for each list of options in
# test_join. This will multiply the number of variants by 2 for set of
# options.
for join_opts in self.test_join:
variants = variants + [opts + join_opts for opts in variants]
# For each list of jit flags, make a copy of the test.
return [self.copy_and_extend_jitflags(v) for v in variants]
COOKIE = b"|jit-test|"
# We would use 500019 (5k19), but quit() only accepts values up to 127, due to fuzzers
SKIPPED_EXIT_STATUS = 59
Directives = {}
@classmethod
def find_directives(cls, file_name):
meta = ""
line = open(file_name, "rb").readline()
i = line.find(cls.COOKIE)
if i != -1:
meta = ";" + line[i + len(cls.COOKIE) :].decode(errors="strict").strip("\n")
return meta
@classmethod
def from_file(cls, path, options):
test = cls(path)
# If directives.txt exists in the test's directory then it may
# contain metainformation that will be catenated with
# whatever's in the test file. The form of the directive in
# the directive file is the same as in the test file. Only
# the first line is considered, just as for the test file.
dir_meta = ""
dir_name = os.path.dirname(path)
if dir_name in cls.Directives:
dir_meta = cls.Directives[dir_name]
else:
meta_file_name = os.path.join(dir_name, "directives.txt")
if os.path.exists(meta_file_name):
dir_meta = cls.find_directives(meta_file_name)
cls.Directives[dir_name] = dir_meta
filename, file_extension = os.path.splitext(path)
meta = cls.find_directives(path)
if meta != "" or dir_meta != "":
meta = meta + dir_meta
parts = meta.split(";")
for part in parts:
part = part.strip()
if not part:
continue
name, _, value = part.partition(":")
if value:
value = value.strip()
if name == "error":
test.expect_error = value
elif name == "exitstatus":
try:
status = int(value, 0)
if status == test.SKIPPED_EXIT_STATUS:
print(
"warning: jit-tests uses {} as a sentinel"
" return value {}",
test.SKIPPED_EXIT_STATUS,
path,
)
else:
test.expect_status = status
except ValueError:
print(
"warning: couldn't parse exit status"
" {}".format(value)
)
elif name == "thread-count":
try:
test.jitflags.append(
"--thread-count={}".format(int(value, 0))
)
except ValueError:
print(
"warning: couldn't parse thread-count"
" {}".format(value)
)
elif name == "include":
test.other_lib_includes.append(value)
elif name == "local-include":
test.other_script_includes.append(value)
elif name == "skip-if":
test.skip_if_cond = extend_condition(test.skip_if_cond, value)
elif name == "skip-variant-if":
try:
[variant, condition] = value.split(",")
test.skip_variant_if_cond[variant] = extend_condition(
test.skip_if_cond, condition
)
except ValueError:
print("warning: couldn't parse skip-variant-if")
else:
print(
"{}: warning: unrecognized |jit-test| attribute"
" {}".format(path, part)
)
else:
if name == "slow":
test.slow = True
elif name == "allow-oom":
test.allow_oom = True
elif name == "allow-unhandlable-oom":
test.allow_unhandlable_oom = True
elif name == "allow-overrecursed":
test.allow_overrecursed = True
elif name == "valgrind":
test.valgrind = options.valgrind
elif name == "tz-pacific":
test.tz_pacific = True
elif name.startswith("test-also="):
test.test_also.append(
re.split(r"\s+", name[len("test-also=") :])
)
elif name.startswith("test-join="):
test.test_join.append(
re.split(r"\s+", name[len("test-join=") :])
)
elif name == "module":
test.is_module = True
elif name == "crash":
# Crashes are only allowed in self-test, as it is
# intended to verify that our testing infrastructure
# works, and not meant as a way to accept temporary
# failing tests. These tests should either be fixed or
# skipped.
assert (
"self-test" in path
), "{}: has an unexpected crash annotation.".format(path)
test.expect_crash = True
elif name.startswith("--"):
# // |jit-test| --ion-gvn=off; --no-sse4
test.jitflags.append(name)
elif name.startswith("-P"):
prefAndValue = name.split()
assert (
len(prefAndValue) == 2
), f"{name}: failed to parse preference"
# // |jit-test| -P pref(=value)?
test.jitflags.append("--setpref=" + prefAndValue[1])
else:
print(
"{}: warning: unrecognized |jit-test| attribute"
" {}".format(path, part)
)
if options.valgrind_all:
test.valgrind = True
if options.test_reflect_stringify is not None:
test.expect_error = ""
test.expect_status = 0
return test
def command(self, prefix, libdir, moduledir, tempdir, remote_prefix=None):
path = self.path
if remote_prefix:
path = self.path.replace(TEST_DIR, remote_prefix)
scriptdir_var = os.path.dirname(path)
if not scriptdir_var.endswith("/"):
scriptdir_var += "/"
# Note: The tempdir provided as argument is managed by the caller
# should remain alive as long as the test harness. Therefore, the XDR
# content of the self-hosted code would be accessible to all JS Shell
# instances.
self.selfhosted_xdr_path = os.path.join(tempdir, SHELL_XDR)
# Platforms where subprocess immediately invokes exec do not care
# whether we use double or single quotes. On windows and when using
# a remote device, however, we have to be careful to use the quote
# style that is the opposite of what the exec wrapper uses.
if remote_prefix:
quotechar = '"'
else:
quotechar = "'"
# Don't merge the expressions: We want separate -e arguments to avoid
# semicolons in the command line, bug 1351607.
exprs = [
"const platform={}".format(js_quote(quotechar, sys.platform)),
"const libdir={}".format(js_quote(quotechar, libdir)),
"const scriptdir={}".format(js_quote(quotechar, scriptdir_var)),
]
# We may have specified '-a' or '-d' twice: once via --jitflags, once
# via the "|jit-test|" line. Remove dups because they are toggles.
# Note: |dict.fromkeys(flags)| is similar to |set(flags)| but it
# preserves order.
cmd = prefix + []
cmd += list(dict.fromkeys(self.jitflags))
# Handle selfhosted XDR file.
if self.selfhosted_xdr_mode != "off":
cmd += [
"--selfhosted-xdr-path",
self.selfhosted_xdr_path,
"--selfhosted-xdr-mode",
self.selfhosted_xdr_mode,
]
for expr in exprs:
cmd += ["-e", expr]
for inc in self.other_lib_includes:
cmd += ["-f", libdir + inc]
for inc in self.other_script_includes:
cmd += ["-f", scriptdir_var + inc]
if self.skip_if_cond:
cmd += [
"-e",
"if ({}) quit({})".format(self.skip_if_cond, self.SKIPPED_EXIT_STATUS),
]
cmd += ["--module-load-path", moduledir]
if self.is_module:
cmd += ["--module", path]
elif self.test_reflect_stringify is None:
cmd += ["-f", path]
else:
cmd += ["--", self.test_reflect_stringify, "--check", path]
if self.valgrind:
cmd = self.VALGRIND_CMD + cmd
if self.allow_unhandlable_oom or self.expect_crash:
cmd += ["--suppress-minidump"]
return cmd
# The test runner expects this to be set to give to get_command.
js_cmd_prefix = None
def get_command(self, prefix, tempdir):
"""Shim for the test runner."""
return self.command(prefix, LIB_DIR, MODULE_DIR, tempdir)
def find_tests(substring=None):
ans = []
for dirpath, dirnames, filenames in os.walk(TEST_DIR):
dirnames.sort()
filenames.sort()
if dirpath == ".":
continue
for filename in filenames:
if not filename.endswith(".js"):
continue
if filename in ("shell.js", "browser.js"):
continue
test = os.path.join(dirpath, filename)
if substring is None or substring in os.path.relpath(test, TEST_DIR):
ans.append(test)
return ans
def check_output(out, err, rc, timed_out, test, options):
# Allow skipping to compose with other expected results
if test.skip_if_cond:
if rc == test.SKIPPED_EXIT_STATUS:
return True
if timed_out:
relpath = os.path.normpath(test.relpath_tests).replace(os.sep, "/")
if relpath in options.ignore_timeouts:
return True
return False
if test.expect_error:
# The shell exits with code 3 on uncaught exceptions.
if rc != 3:
return False
return test.expect_error in err
for line in out.split("\n"):
if line.startswith("Trace stats check failed"):
return False
for line in err.split("\n"):
if "Assertion failed:" in line:
return False
if test.expect_crash:
# Python 3 on Windows interprets process exit codes as unsigned
# integers, where Python 2 used to allow signed integers. Account for
# each possibility here.
if sys.platform == "win32" and rc in (3 - 2**31, 3 + 2**31):
return True
if sys.platform != "win32" and rc == -11:
return True
# When building with ASan enabled, ASan will convert the -11 returned
# value to 1. As a work-around we look for the error output which
# includes the crash reason.
if rc == 1 and ("Hit MOZ_CRASH" in err or "Assertion failure:" in err):
return True
# When running jittests on Android, SEGV results in a return code of
# 128 + 11 = 139. Due to a bug in tinybox, we have to check for 138 as
# well.
if rc == 139 or rc == 138:
return True
# Crashing test should always crash as expected, otherwise this is an
# error. The JS shell crash() function can be used to force the test
# case to crash in unexpected configurations.
return False
if rc != test.expect_status:
# Allow a non-zero exit code if we want to allow OOM, but only if we
# actually got OOM.
if (
test.allow_oom
and "out of memory" in err
and "Assertion failure" not in err
and "MOZ_CRASH" not in err
):
return True
# Allow a non-zero exit code if we want to allow unhandlable OOM, but
# only if we actually got unhandlable OOM.
if test.allow_unhandlable_oom and "MOZ_CRASH([unhandlable oom]" in err:
return True
# Allow a non-zero exit code if we want to all too-much-recursion and
# the test actually over-recursed.
if (
test.allow_overrecursed
and "too much recursion" in err
and "Assertion failure" not in err
):
return True
# Allow a zero exit code if we are running under a sanitizer that
# forces the exit status.
if test.expect_status != 0 and options.unusable_error_status:
return True
return False
return True
def print_automation_format(ok, res, slog):
# Output test failures in a parsable format suitable for automation, eg:
# TEST-RESULT | filename.js | Failure description (code N, args "--foobar")
#
# Example:
# TEST-PASS | foo/bar/baz.js | (code 0, args "--ion-eager")
# TEST-UNEXPECTED-FAIL | foo/bar/baz.js | TypeError: or something (code -9, args "--no-ion")
# INFO exit-status : 3
# INFO timed-out : False
# INFO stdout > foo
# INFO stdout > bar
# INFO stdout > baz
# INFO stderr 2> TypeError: or something
# TEST-UNEXPECTED-FAIL | jit_test.py: Test execution interrupted by user
result = "TEST-PASS" if ok else "TEST-UNEXPECTED-FAIL"
message = "Success" if ok else res.describe_failure()
jitflags = " ".join(res.test.jitflags)
print(
'{} | {} | {} (code {}, args "{}") [{:.1f} s]'.format(
result, res.test.relpath_top, message, res.rc, jitflags, res.dt
)
)
details = {
"message": message,
"extra": {
"jitflags": jitflags,
},
}
if res.extra:
details["extra"].update(res.extra)
slog.test(res.test.relpath_tests, "PASS" if ok else "FAIL", res.dt, **details)
# For failed tests, print as much information as we have, to aid debugging.
if ok:
return
print("INFO exit-status : {}".format(res.rc))
print("INFO timed-out : {}".format(res.timed_out))
for line in res.out.splitlines():
print("INFO stdout > " + line.strip())
warnings = []
for line in res.err.splitlines():
# See Bug 1868693
if line.startswith("WARNING") and "unused DT entry" in line:
warnings.append(line)
else:
print("INFO stderr 2> " + line.strip())
for line in warnings:
print("INFO (warn-stderr) 2> " + line.strip())
def print_test_summary(num_tests, failures, complete, slow_tests, doing, options):
def test_details(res):
if options.show_failed:
return escape_cmdline(res.cmd)
return " ".join(res.test.jitflags + [res.test.relpath_tests])
if failures:
if options.write_failures:
try:
out = open(options.write_failures, "w")
# Don't write duplicate entries when we are doing multiple
# failures per job.
written = set()
for res in failures:
if res.test.path not in written:
out.write(os.path.relpath(res.test.path, TEST_DIR) + "\n")
if options.write_failure_output:
out.write(res.out)
out.write(res.err)
out.write("Exit code: " + str(res.rc) + "\n")
written.add(res.test.path)
out.close()
except IOError:
sys.stderr.write(
"Exception thrown trying to write failure"
" file '{}'\n".format(options.write_failures)
)
traceback.print_exc()
sys.stderr.write("---\n")
print("FAILURES:")
for res in failures:
if not res.timed_out:
print(" " + test_details(res))
print("TIMEOUTS:")
for res in failures:
if res.timed_out:
print(" " + test_details(res))
else:
print(
"PASSED ALL"
+ (
""
if complete
else " (partial run -- interrupted by user {})".format(doing)
)
)
if options.format == "automation":
num_failures = len(failures) if failures else 0
print("Result summary:")
print("Passed: {:d}".format(num_tests - num_failures))
print("Failed: {:d}".format(num_failures))
if num_tests != 0 and options.show_slow:
threshold = options.slow_test_threshold
fraction_fast = 1 - len(slow_tests) / num_tests
print(
"{:5.2f}% of tests ran in under {}s".format(fraction_fast * 100, threshold)
)
print("Slowest tests that took longer than {}s:".format(threshold))
slow_tests.sort(key=lambda res: res.dt, reverse=True)
any = False
for i in range(min(len(slow_tests), 20)):
res = slow_tests[i]
print(" {:6.2f} {}".format(res.dt, test_details(res)))
any = True
if not any:
print("None")
return not failures
def create_progressbar(num_tests, options):
if (
not options.hide_progress
and not options.show_cmd
and ProgressBar.conservative_isatty()
):
fmt = [
{"value": "PASS", "color": "green"},
{"value": "FAIL", "color": "red"},
{"value": "TIMEOUT", "color": "blue"},
{"value": "SKIP", "color": "brightgray"},
]
return ProgressBar(num_tests, fmt)
return NullProgressBar()
def process_test_results(results, num_tests, pb, options, slog):
failures = []
timeouts = 0
complete = False
output_dict = {}
doing = "before starting"
slow_tests = []
if num_tests == 0:
pb.finish(True)
complete = True
return print_test_summary(
num_tests, failures, complete, slow_tests, doing, options
)
try:
for i, res in enumerate(results):
ok = check_output(
res.out, res.err, res.rc, res.timed_out, res.test, options
)
if ok:
show_output = options.show_output and not options.failed_only
else:
show_output = options.show_output or not options.no_show_failed
if show_output:
pb.beginline()
sys.stdout.write(res.out)
sys.stdout.write(res.err)
sys.stdout.write("Exit code: {}\n".format(res.rc))
if res.test.valgrind and not show_output:
pb.beginline()
sys.stdout.write(res.err)
if options.check_output:
if res.test.path in output_dict.keys():
if output_dict[res.test.path] != res.out:
pb.message(
"FAIL - OUTPUT DIFFERS {}".format(res.test.relpath_tests)
)
else:
output_dict[res.test.path] = res.out
doing = "after {}".format(res.test.relpath_tests)
if not ok:
failures.append(res)
if res.timed_out:
pb.message("TIMEOUT - {}".format(res.test.relpath_tests))
timeouts += 1
else:
pb.message("FAIL - {}".format(res.test.relpath_tests))
if options.format == "automation":
print_automation_format(ok, res, slog)
n = i + 1
pb.update(
n,
{
"PASS": n - len(failures),
"FAIL": len(failures),
"TIMEOUT": timeouts,
"SKIP": 0,
},
)
if res.dt > options.slow_test_threshold:
slow_tests.append(res)
complete = True
except KeyboardInterrupt:
print(
"TEST-UNEXPECTED-FAIL | jit_test.py"
+ " : Test execution interrupted by user"
)
pb.finish(True)
return print_test_summary(num_tests, failures, complete, slow_tests, doing, options)
def run_tests(tests, num_tests, prefix, options, remote=False):
slog = None
if options.format == "automation":
slog = TestLogger("jittests")
slog.suite_start()
if remote:
ok = run_tests_remote(tests, num_tests, prefix, options, slog)
else:
ok = run_tests_local(tests, num_tests, prefix, options, slog)
if slog:
slog.suite_end()
return ok
def run_tests_local(tests, num_tests, prefix, options, slog):
# The jstests tasks runner requires the following options. The names are
# taken from the jstests options processing code, which are frequently
# subtly different from the options jit-tests expects. As such, we wrap
# them here, as needed.
AdaptorOptions = namedtuple(
"AdaptorOptions",
[
"worker_count",
"passthrough",
"timeout",
"output_fp",
"hide_progress",
"run_skipped",
"show_cmd",
"use_xdr",
],
)
shim_options = AdaptorOptions(
options.max_jobs,
False,
options.timeout,
sys.stdout,
False,
True,
options.show_cmd,
options.use_xdr,
)
# The test runner wants the prefix as a static on the Test class.
JitTest.js_cmd_prefix = prefix
with TemporaryDirectory() as tempdir:
pb = create_progressbar(num_tests, options)
gen = run_all_tests(tests, prefix, tempdir, pb, shim_options)
ok = process_test_results(gen, num_tests, pb, options, slog)
return ok
def run_tests_remote(tests, num_tests, prefix, options, slog):
# Setup device with everything needed to run our tests.
from mozdevice import ADBError, ADBTimeoutError
from .tasks_adb_remote import get_remote_results
# Run all tests.
pb = create_progressbar(num_tests, options)
try:
gen = get_remote_results(tests, prefix, pb, options)
ok = process_test_results(gen, num_tests, pb, options, slog)
except (ADBError, ADBTimeoutError):
print("TEST-UNEXPECTED-FAIL | jit_test.py" + " : Device error during test")
raise
return ok
if __name__ == "__main__":
print("Use ../jit-test/jit_test.py to run these tests.")
|