1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
|
/**
* By default this test only tests 1 sample. This is to avoid publishing all samples we have
* to the codebase. If you update the Fathom CC model, please follow the instruction below
* and run the test. Doing this makes sure the optimized (Native implementation) CC fathom model produces
* exactly the same result as the non-optimized model (JS implementation, See CreditCardRuleset.sys.mjs).
*
* To test this:
* 1. Run the test setup script (fathom/test-setup.sh) to download all samples to the local
* directory. Note that you need to have the permission to access the fathom-form-autofill
* 2. Set `gTestAutofillRepoSample` to true
* 3. Run this test
*/
"use strict";
const eligibleElementSelector =
"input:not([type]), input[type=text], input[type=textbox], input[type=email], input[type=tel], input[type=number], input[type=month], select, button";
const skippedSamples = [
// TOOD: Crash while running the following testcases. Since this is not caused by the fathom CC
// model, we just skip those for now
"EN_B105b.html",
"EN_B312a.html",
"EN_B118b.html",
"EN_B48c.html",
// This sample is skipped because of Bug 1754256 (Support lookaround regex for native fathom CC implementation).
"DE_B378b.html",
];
async function run_test(path, dirs) {
await SpecialPowers.pushPrefEnv({
set: [["extensions.formautofill.creditCards.heuristics.mode", 1]],
});
// Collect files we are going to test.
let files = [];
for (let dir of dirs) {
let entries = new FileUtils.File(getTestFilePath(path + dir))
.directoryEntries;
while (entries.hasMoreElements()) {
let entry = entries.nextFile;
if (skippedSamples.includes(entry.leafName)) {
continue;
}
if (entry.leafName.endsWith(".html")) {
files.push(path + dir + entry.leafName);
}
}
}
ok(files.length, "no sample files found");
let summary = {};
for (let file of files) {
info("Testing " + file + "...");
await BrowserTestUtils.withNewTab(BASE_URL + file, async browser => {
summary[file] = await SpecialPowers.spawn(
browser,
[{ eligibleElementSelector, file }],
obj => {
const { FormAutofillHeuristics } = ChromeUtils.importESModule(
"resource://gre/modules/shared/FormAutofillHeuristics.sys.mjs"
);
const { FormAutofillUtils } = ChromeUtils.importESModule(
"resource://gre/modules/shared/FormAutofillUtils.sys.mjs"
);
let eligibleFields = [];
let nodeList = content.document.querySelectorAll(
obj.eligibleElementSelector
);
for (let i = 0; i < nodeList.length; i++) {
if (FormAutofillUtils.isCreditCardOrAddressFieldType(nodeList[i])) {
eligibleFields.push(nodeList[i]);
}
}
let failedFields = [];
info("Running CC fathom model");
let nativeConfidencesKeyedByType =
ChromeUtils.getFormAutofillConfidences(eligibleFields);
let jsConfidencesKeyedByType =
FormAutofillHeuristics.getFormAutofillConfidences(eligibleFields);
if (eligibleFields.length != nativeConfidencesKeyedByType.length) {
ok(
false,
`Get the wrong number of confidence value from the native model`
);
}
if (eligibleFields.length != jsConfidencesKeyedByType.length) {
ok(
false,
`Get the wrong number of confidence value from the js model`
);
}
// This value should sync with the number of supported types in
// CreditCardRuleset.sys.mjs (See `get types()` in `this.creditCardRulesets`).
const EXPECTED_NUM_OF_CONFIDENCE = 2;
for (let i = 0; i < eligibleFields.length; i++) {
if (
Object.keys(nativeConfidencesKeyedByType[i]).length !=
EXPECTED_NUM_OF_CONFIDENCE
) {
ok(
false,
`Native CC model doesn't get confidence value for all types`
);
}
if (
Object.keys(jsConfidencesKeyedByType[i]).length !=
EXPECTED_NUM_OF_CONFIDENCE
) {
ok(
false,
`JS CC model doesn't get confidence value for all types`
);
}
for (let [type, confidence] of Object.entries(
nativeConfidencesKeyedByType[i]
)) {
// Fix to 10 digit to ignore rounding error between js and c++.
let nativeConfidence = confidence.toFixed(10);
let jsConfidence =
jsConfidencesKeyedByType[i][
FormAutofillUtils.formAutofillConfidencesKeyToCCFieldType(
type
)
].toFixed(10);
if (jsConfidence != nativeConfidence) {
info(
`${obj.file}: Element(id=${eligibleFields[i].id} doesn't have the same confidence value when rule type is ${type}`
);
if (!failedFields.includes(i)) {
failedFields.push(i);
}
}
}
}
ok(
!failedFields.length,
`${obj.file}: has the same confidences value on both models`
);
return {
tested: eligibleFields.length,
passed: eligibleFields.length - failedFields.length,
};
}
);
});
}
// Generating summary report
let total_tested_samples = 0;
let total_passed_samples = 0;
let total_tested_fields = 0;
let total_passed_fields = 0;
info("=====Summary=====");
for (const [k, v] of Object.entries(summary)) {
total_tested_samples++;
if (v.tested == v.passed) {
total_passed_samples++;
} else {
info("Failed Case:" + k);
}
total_tested_fields += v.tested;
total_passed_fields += v.passed;
}
info(
"Passed Samples/Test Samples: " +
total_passed_samples +
"/" +
total_tested_samples
);
info(
"Passed Fields/Test Fields: " +
total_passed_fields +
"/" +
total_tested_fields
);
}
add_task(async function test_native_cc_model() {
const path = "fathom/";
const dirs = ["testing/"];
await run_test(path, dirs);
});
add_task(async function test_native_cc_model_autofill_repo() {
const path = "fathom/autofill-repo-samples/";
const dirs = ["validation/", "training/", "testing/"];
if (await IOUtils.exists(getTestFilePath(path))) {
// Just to ignore timeout failure while running the test on the local
requestLongerTimeout(10);
await run_test(path, dirs);
}
});
|
