blob: 9731d40cb6046b42e48f9fb7249343b4286f289f (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
|
#!/usr/bin/env bash
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# This script does the following:
# 1. Retrieves glcoud service account token
# 2. Activates gcloud service account
# 3. Connects to google Firebase (using TestArmada's Flank tool)
# 4. Executes UI tests
# 5. Puts test artifacts into the test_artifacts folder
# NOTE:
# Flank supports sharding across multiple devices at a time, but gcloud API
# only supports 1 defined APK per test run.
# If a command fails then do not proceed and fail this script too.
set -e
#########################
# The command line help #
#########################
display_help() {
echo "Usage: $0 Component_Name Build_Variant [Number_Shards...]"
echo
echo "Examples:"
echo "To run component/browser tests on ARM device shard (1 test / shard)"
echo "$ execute-firebase-test.sh component arm"
echo
echo "To run component/feature tests on X86 device (on 3 shards)"
echo "$ execute-firebase-test.sh feature x86 3"
echo
echo "To run UI samples/sampleName tests"
echo "$ execute-firebase-test.sh sample-sampleName arm 1"
echo
}
# Basic parameter check
if [[ $# -lt 2 ]]; then
echo "Your command line contains $# arguments"
display_help
exit 1
fi
component="$1" # browser, concept, feature
device_type="$2" # arm | x86
if [[ ! -z "$3" ]]; then
num_shards=$3
fi
JAVA_BIN="/usr/bin/java"
PATH_TEST="./automation/taskcluster/androidTest"
FLANK_BIN="/builds/worker/test-tools/flank.jar"
FLANK_CONF_ARM="${PATH_TEST}/flank-arm.yml"
FLANK_CONF_X86="${PATH_TEST}/flank-x86.yml"
ARTIFACT_DIR="/builds/worker/artifacts"
RESULTS_DIR="${ARTIFACT_DIR}/results"
echo
echo "ACTIVATE SERVICE ACCT"
echo
# this is where the Google Testcloud project ID is set
gcloud config set project "$GOOGLE_PROJECT"
echo
gcloud auth activate-service-account --key-file "$GOOGLE_APPLICATION_CREDENTIALS"
echo
echo
# From now on disable exiting on error. If the tests fail we want to continue
# and try to download the artifacts. We will exit with the actual error code later.
set +e
if [[ "${device_type,,}" == "x86" ]]
then
flank_template="$FLANK_CONF_X86"
else
flank_template="$FLANK_CONF_ARM"
fi
# Remove samples- from the component for each APK path
samples=${component//samples-}
# If tests are for components, the path is different than for samples
if [[ "${component}" != samples-* ]]
then
# Case 1: tests for any component (but NOT samples, NOT real UI tests)
APK_APP="./samples/browser/build/outputs/apk/gecko/debug/samples-browser-gecko-debug.apk"
if [[ "${component}" == *"-"* ]]
then
regex='([a-z]*)-(.*)'
[[ "$component" =~ $regex ]]
APK_TEST="./components/${BASH_REMATCH[1]}/${BASH_REMATCH[2]}/build/outputs/apk/androidTest/debug/${component}-debug-androidTest.apk"
else
APK_TEST="./components/${component}/engine-gecko/build/outputs/apk/androidTest/debug/browser-engine-gecko-debug-androidTest.apk"
fi
elif [[ "${component}" == "samples-browser" ]]
then
# Case 2: tests for browser sample (gecko sample only)
APK_APP="./samples/${samples}/build/outputs/apk/gecko/debug/samples-${samples}-gecko-debug.apk"
APK_TEST="./samples/${samples}/build/outputs/apk/androidTest/gecko/debug/samples-{$samples}-gecko-debug-androidTest.apk"
else
# Case 3: tests for non-browser samples (i.e. samples-glean)
APK_APP="./samples/${samples}/build/outputs/apk/debug/samples-${samples}-debug.apk"
APK_TEST="./samples/${samples}/build/outputs/apk/androidTest/debug/samples-${samples}-debug-androidTest.apk"
fi
# function to exit script with exit code from test run.
# (Only 0 if all test executions passed)
function failure_check() {
echo
echo
if [[ $exitcode -ne 0 ]]; then
echo "ERROR: UI test run failed, please check above URL"
else
echo "All UI test(s) have passed!"
fi
echo
echo "RESULTS"
echo
mkdir -p ${ARTIFACT_DIR}/github
chmod +x ${PATH_TEST}/parse-ui-test.py
${PATH_TEST}/parse-ui-test.py \
--exit-code "${exitcode}" \
--log flank.log \
--results "${RESULTS_DIR}" \
--output-md "${ARTIFACT_DIR}/github/customCheckRunText.md" \
--device-type "${device_type}"
echo
echo
}
echo
echo "EXECUTE TEST(S)"
echo
set -o pipefail && $JAVA_BIN -jar $FLANK_BIN android run \
--config=$flank_template \
--max-test-shards=$num_shards \
--app=$APK_APP --test=$APK_TEST \
--local-result-dir="${RESULTS_DIR}" \
--project=$GOOGLE_PROJECT \
--client-details=commit=${MOBILE_HEAD_REV:-None},pullRequest=${PULL_REQUEST_NUMBER:-None} \
| tee flank.log
exitcode=$?
failure_check
exit $exitcode
|
