1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
|
# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import collections
import json
import logging
import multiprocessing
import os
import select
import subprocess
import sys
import zipfile
from six.moves import range # pylint: disable=redefined-builtin
from pylib import constants
from pylib.base import base_test_result
from pylib.base import test_run
from pylib.constants import host_paths
from pylib.results import json_results
from py_utils import tempfile_ext
# These Test classes are used for running tests and are excluded in the test
# runner. See:
# https://android.googlesource.com/platform/frameworks/testing/+/android-support-test/runner/src/main/java/android/support/test/internal/runner/TestRequestBuilder.java
# base/test/android/javatests/src/org/chromium/base/test/BaseChromiumAndroidJUnitRunner.java # pylint: disable=line-too-long
_EXCLUDED_CLASSES_PREFIXES = ('android', 'junit', 'org/bouncycastle/util',
'org/hamcrest', 'org/junit', 'org/mockito')
# Suites we shouldn't shard, usually because they don't contain enough test
# cases.
_EXCLUDED_SUITES = {
'password_check_junit_tests',
'touch_to_fill_junit_tests',
}
# It can actually take longer to run if you shard too much, especially on
# smaller suites. Locally media_base_junit_tests takes 4.3 sec with 1 shard,
# and 6 sec with 2 or more shards.
_MIN_CLASSES_PER_SHARD = 8
class LocalMachineJunitTestRun(test_run.TestRun):
def __init__(self, env, test_instance):
super(LocalMachineJunitTestRun, self).__init__(env, test_instance)
#override
def TestPackage(self):
return self._test_instance.suite
#override
def SetUp(self):
pass
def _CreateJarArgsList(self, json_result_file_paths, group_test_list, shards):
# Creates a list of jar_args. The important thing is each jar_args list
# has a different json_results file for writing test results to and that
# each list of jar_args has its own test to run as specified in the
# -gtest-filter.
jar_args_list = [['-json-results-file', result_file]
for result_file in json_result_file_paths]
for index, jar_arg in enumerate(jar_args_list):
if shards > 1:
jar_arg.extend(['-gtest-filter', ':'.join(group_test_list[index])])
elif self._test_instance.test_filter:
jar_arg.extend(['-gtest-filter', self._test_instance.test_filter])
if self._test_instance.package_filter:
jar_arg.extend(['-package-filter', self._test_instance.package_filter])
if self._test_instance.runner_filter:
jar_arg.extend(['-runner-filter', self._test_instance.runner_filter])
return jar_args_list
def _CreateJvmArgsList(self):
# Creates a list of jvm_args (robolectric, code coverage, etc...)
jvm_args = [
'-Drobolectric.dependency.dir=%s' %
self._test_instance.robolectric_runtime_deps_dir,
'-Ddir.source.root=%s' % constants.DIR_SOURCE_ROOT,
'-Drobolectric.resourcesMode=binary',
]
if logging.getLogger().isEnabledFor(logging.INFO):
jvm_args += ['-Drobolectric.logging=stdout']
if self._test_instance.debug_socket:
jvm_args += [
'-agentlib:jdwp=transport=dt_socket'
',server=y,suspend=y,address=%s' % self._test_instance.debug_socket
]
if self._test_instance.coverage_dir:
if not os.path.exists(self._test_instance.coverage_dir):
os.makedirs(self._test_instance.coverage_dir)
elif not os.path.isdir(self._test_instance.coverage_dir):
raise Exception('--coverage-dir takes a directory, not file path.')
if self._test_instance.coverage_on_the_fly:
jacoco_coverage_file = os.path.join(
self._test_instance.coverage_dir,
'%s.exec' % self._test_instance.suite)
jacoco_agent_path = os.path.join(host_paths.DIR_SOURCE_ROOT,
'third_party', 'jacoco', 'lib',
'jacocoagent.jar')
# inclnolocationclasses is false to prevent no class def found error.
jacoco_args = '-javaagent:{}=destfile={},inclnolocationclasses=false'
jvm_args.append(
jacoco_args.format(jacoco_agent_path, jacoco_coverage_file))
else:
jvm_args.append('-Djacoco-agent.destfile=%s' %
os.path.join(self._test_instance.coverage_dir,
'%s.exec' % self._test_instance.suite))
return jvm_args
#override
def RunTests(self, results):
wrapper_path = os.path.join(constants.GetOutDirectory(), 'bin', 'helper',
self._test_instance.suite)
# This avoids searching through the classparth jars for tests classes,
# which takes about 1-2 seconds.
# Do not shard when a test filter is present since we do not know at this
# point which tests will be filtered out.
if (self._test_instance.shards == 1 or self._test_instance.test_filter
or self._test_instance.suite in _EXCLUDED_SUITES):
test_classes = []
shards = 1
else:
test_classes = _GetTestClasses(wrapper_path)
shards = ChooseNumOfShards(test_classes, self._test_instance.shards)
logging.info('Running tests on %d shard(s).', shards)
group_test_list = GroupTestsForShard(shards, test_classes)
with tempfile_ext.NamedTemporaryDirectory() as temp_dir:
cmd_list = [[wrapper_path] for _ in range(shards)]
json_result_file_paths = [
os.path.join(temp_dir, 'results%d.json' % i) for i in range(shards)
]
jar_args_list = self._CreateJarArgsList(json_result_file_paths,
group_test_list, shards)
for i in range(shards):
cmd_list[i].extend(['--jar-args', '"%s"' % ' '.join(jar_args_list[i])])
jvm_args = self._CreateJvmArgsList()
if jvm_args:
for cmd in cmd_list:
cmd.extend(['--jvm-args', '"%s"' % ' '.join(jvm_args)])
AddPropertiesJar(cmd_list, temp_dir, self._test_instance.resource_apk)
procs = [
subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT) for cmd in cmd_list
]
PrintProcessesStdout(procs)
results_list = []
try:
for json_file_path in json_result_file_paths:
with open(json_file_path, 'r') as f:
results_list += json_results.ParseResultsFromJson(
json.loads(f.read()))
except IOError:
# In the case of a failure in the JUnit or Robolectric test runner
# the output json file may never be written.
results_list = [
base_test_result.BaseTestResult(
'Test Runner Failure', base_test_result.ResultType.UNKNOWN)
]
test_run_results = base_test_result.TestRunResults()
test_run_results.AddResults(results_list)
results.append(test_run_results)
#override
def TearDown(self):
pass
def AddPropertiesJar(cmd_list, temp_dir, resource_apk):
# Create properties file for Robolectric test runners so they can find the
# binary resources.
properties_jar_path = os.path.join(temp_dir, 'properties.jar')
with zipfile.ZipFile(properties_jar_path, 'w') as z:
z.writestr('com/android/tools/test_config.properties',
'android_resource_apk=%s' % resource_apk)
for cmd in cmd_list:
cmd.extend(['--classpath', properties_jar_path])
def ChooseNumOfShards(test_classes, shards):
# Don't override requests to not shard.
if shards == 1:
return 1
# Sharding doesn't reduce runtime on just a few tests.
if shards > (len(test_classes) // _MIN_CLASSES_PER_SHARD) or shards < 1:
shards = max(1, (len(test_classes) // _MIN_CLASSES_PER_SHARD))
# Local tests of explicit --shard values show that max speed is achieved
# at cpu_count() / 2.
# Using -XX:TieredStopAtLevel=1 is required for this result. The flag reduces
# CPU time by two-thirds, making sharding more effective.
shards = max(1, min(shards, multiprocessing.cpu_count() // 2))
# Can have at minimum one test_class per shard.
shards = min(len(test_classes), shards)
return shards
def GroupTestsForShard(num_of_shards, test_classes):
"""Groups tests that will be ran on each shard.
Args:
num_of_shards: number of shards to split tests between.
test_classes: A list of test_class files in the jar.
Return:
Returns a dictionary containing a list of test classes.
"""
test_dict = {i: [] for i in range(num_of_shards)}
# Round robin test distribiution to reduce chance that a sequential group of
# classes all have an unusually high number of tests.
for count, test_cls in enumerate(test_classes):
test_cls = test_cls.replace('.class', '*')
test_cls = test_cls.replace('/', '.')
test_dict[count % num_of_shards].append(test_cls)
return test_dict
def PrintProcessesStdout(procs):
"""Prints the stdout of all the processes.
Buffers the stdout of the processes and prints it when finished.
Args:
procs: A list of subprocesses.
Returns: N/A
"""
streams = [p.stdout for p in procs]
outputs = collections.defaultdict(list)
first_fd = streams[0].fileno()
while streams:
rstreams, _, _ = select.select(streams, [], [])
for stream in rstreams:
line = stream.readline()
if line:
# Print out just one output so user can see work being done rather
# than waiting for it all at the end.
if stream.fileno() == first_fd:
sys.stdout.write(line)
else:
outputs[stream.fileno()].append(line)
else:
streams.remove(stream) # End of stream.
for p in procs:
sys.stdout.write(''.join(outputs[p.stdout.fileno()]))
def _GetTestClasses(file_path):
test_jar_paths = subprocess.check_output([file_path, '--print-classpath'])
test_jar_paths = test_jar_paths.split(':')
test_classes = []
for test_jar_path in test_jar_paths:
# Avoid searching through jars that are for the test runner.
# TODO(crbug.com/1144077): Use robolectric buildconfig file arg.
if 'third_party/robolectric/' in test_jar_path:
continue
test_classes += _GetTestClassesFromJar(test_jar_path)
logging.info('Found %d test classes in class_path jars.', len(test_classes))
return test_classes
def _GetTestClassesFromJar(test_jar_path):
"""Returns a list of test classes from a jar.
Test files end in Test, this is enforced:
//tools/android/errorprone_plugin/src/org/chromium/tools/errorprone
/plugin/TestClassNameCheck.java
Args:
test_jar_path: Path to the jar.
Return:
Returns a list of test classes that were in the jar.
"""
class_list = []
with zipfile.ZipFile(test_jar_path, 'r') as zip_f:
for test_class in zip_f.namelist():
if test_class.startswith(_EXCLUDED_CLASSES_PREFIXES):
continue
if test_class.endswith('Test.class') and '$' not in test_class:
class_list.append(test_class)
return class_list
|