1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
|
/* Any copyright is dedicated to the Public Domain.
http://creativecommons.org/publicdomain/zero/1.0/ */
"use strict";
/**
* @typedef {import("../translations").RemoteSettingsClient} RemoteSettingsClient
* @typedef {import("../../translations").TranslationModelRecord} TranslationModelRecord
*/
const { RemoteSettings } = ChromeUtils.importESModule(
"resource://services-settings/remote-settings.sys.mjs"
);
// The full Firefox version string.
const firefoxFullVersion = AppConstants.MOZ_APP_VERSION_DISPLAY;
// The Firefox major version string (i.e. the first set of digits).
const firefoxMajorVersion = firefoxFullVersion.match(/\d+/);
// The Firefox "AlphaZero" version string.
// This is a version that is less than even the latest Nightly
// which is of the form `${firefoxMajorVersion}.a1`.
const firefoxAlphaZeroVersion = `${firefoxMajorVersion}.a0`;
/**
* Creates a local RemoteSettingsClient for use within tests.
*
* @param {string} mockedKey
* @returns {RemoteSettingsClient}
*/
async function createRemoteSettingsClient(mockedKey) {
const client = RemoteSettings(mockedKey);
await client.db.clear();
await client.db.importChanges({}, Date.now());
return client;
}
// The following test ensures the capabilities of `filter_expression` in remote settings
// to successfully discriminate against the Firefox version when retrieving records.
//
// This is used when making major breaking changes that would require particular records
// to only show up in certain versions of Firefox, such as actual code changes that no
// longer allow compatibility with given records.
//
// Some examples might be:
//
// - Modifying a wasm.js file that is no longer compatible with a previous wasm binary record.
// In such a case, the old binary record would need to be shipped in versions with the old
// wasm.js file, and the new binary record would need to be shipped in version with the new
// wasm.js file.
//
// - Switching to a different library for translation or language detection.
// Using a different library would not only mean major changes to the code, but it would
// certainly mean that the records for the old libraries are no longer compatible.
// We will need to ship those records only in older versions of Firefox that utilize the old
// libraries, and ship new records in the new versions of Firefox.
add_task(async function test_filter_current_firefox_version() {
// Create a new RemoteSettingsClient just for this test.
let client = await createRemoteSettingsClient(
"test_filter_current_firefox_version"
);
// Create a list of records that are expected to pass the filter_expression and be
// successfully retrieved from the remote settings client.
const expectedPresentRecords = [
{
name: `undefined filter expression`,
filter_expression: undefined,
},
{
name: `null filter expression`,
filter_expression: null,
},
{
name: `empty filter expression`,
filter_expression: ``,
},
{
name: `env.version == ${firefoxFullVersion}`,
filter_expression: `env.version|versionCompare('${firefoxFullVersion}') == 0`,
},
{
name: `env.version > ${firefoxAlphaZeroVersion}`,
filter_expression: `env.version|versionCompare('${firefoxAlphaZeroVersion}') > 0`,
},
];
for (let record of expectedPresentRecords) {
client.db.create(record);
}
// Create a list of records that are expected fo fail the filter_expression and be
// absent when we retrieve the records from the remote settings client.
const expectedAbsentRecords = [
{
name: `env.version < 1`,
filter_expression: `env.version|versionCompare('1') < 0`,
},
];
for (let record of expectedAbsentRecords) {
client.db.create(record);
}
const retrievedRecords = await client.get();
// Ensure that each record that is expected to be present exists in the retrieved records.
for (let expectedPresentRecord of expectedPresentRecords) {
is(
retrievedRecords.some(
record => record.name == expectedPresentRecord.name
),
true,
`The following record was expected to be present but was not found: ${expectedPresentRecord.name}\n`
);
}
// Ensure that each record that is expected to be absent does not exist in the retrieved records.
for (let expectedAbsentRecord of expectedAbsentRecords) {
is(
retrievedRecords.some(record => record.name == expectedAbsentRecord.name),
false,
`The following record was expected to be absent but was found: ${expectedAbsentRecord.name}\n`
);
}
// Ensure that the length of the retrieved records is exactly the length of the records expected to be present.
is(
retrievedRecords.length,
expectedPresentRecords.length,
`Expected ${expectedPresentRecords.length} items but got ${retrievedRecords.length} items\n`
);
});
// The following test ensures that we are able to always retrieve the maximum
// compatible version of records. These are for version changes that do not
// require shipping different records based on a particular Firefox version,
// but rather for changes to model content or wasm runtimes that are fully
// compatible with the existing source code.
add_task(async function test_get_records_with_multiple_versions() {
// Create a new RemoteSettingsClient just for this test.
let client = await createRemoteSettingsClient(
"test_get_translation_model_records"
);
const lookupKey = record =>
`${record.name}${TranslationsParent.languagePairKey(
record.fromLang,
record.toLang
)}`;
// A mapping of each record name to its max version.
const maxVersionMap = {};
// Create a list of records that are all version 1.0
/** @type {TranslationModelRecord[]} */
const versionOneRecords = [
{
id: crypto.randomUUID(),
name: "qualityModel.enes.bin",
fromLang: "en",
toLang: "es",
fileType: "qualityModel",
version: "1.0",
},
{
id: crypto.randomUUID(),
name: "vocab.esen.spm",
fromLang: "en",
toLang: "es",
fileType: "vocab",
version: "1.0",
},
{
id: crypto.randomUUID(),
name: "vocab.esen.spm",
fromLang: "es",
toLang: "en",
fileType: "vocab",
version: "1.0",
},
{
id: crypto.randomUUID(),
name: "lex.50.50.enes.s2t.bin",
fromLang: "en",
toLang: "es",
fileType: "lex",
version: "1.0",
},
{
id: crypto.randomUUID(),
name: "model.enes.intgemm.alphas.bin",
fromLang: "en",
toLang: "es",
fileType: "model",
version: "1.0",
},
{
id: crypto.randomUUID(),
name: "vocab.deen.spm",
fromLang: "en",
toLang: "de",
fileType: "vocab",
version: "1.0",
},
{
id: crypto.randomUUID(),
name: "lex.50.50.ende.s2t.bin",
fromLang: "en",
toLang: "de",
fileType: "lex",
version: "1.0",
},
{
id: crypto.randomUUID(),
name: "model.ende.intgemm.alphas.bin",
fromLang: "en",
toLang: "de",
fileType: "model",
version: "1.0",
},
];
versionOneRecords.reduce((map, record) => {
map[lookupKey(record)] = record.version;
return map;
}, maxVersionMap);
for (const record of versionOneRecords) {
client.db.create(record);
}
// Create a list of records that are identical to some of the above, but with higher version numbers.
const higherVersionRecords = [
{
id: crypto.randomUUID(),
name: "qualityModel.enes.bin",
fromLang: "en",
toLang: "es",
fileType: "qualityModel",
version: "1.1",
},
{
id: crypto.randomUUID(),
name: "qualityModel.enes.bin",
fromLang: "en",
toLang: "es",
fileType: "qualityModel",
version: "1.2",
},
{
id: crypto.randomUUID(),
name: "vocab.esen.spm",
fromLang: "en",
toLang: "es",
fileType: "vocab",
version: "1.1",
},
];
higherVersionRecords.reduce((map, record) => {
const key = lookupKey(record);
if (record.version > map[key]) {
map[key] = record.version;
}
return map;
}, maxVersionMap);
for (const record of higherVersionRecords) {
client.db.create(record);
}
TranslationsParent.mockTranslationsEngine(
client,
await createTranslationsWasmRemoteClient()
);
const retrievedRecords = await TranslationsParent.getMaxVersionRecords(
client,
{ lookupKey }
);
for (const record of retrievedRecords) {
is(
lookupKey(record) in maxVersionMap,
true,
`Expected record ${record.name} to be contained in the nameToVersionMap, but found none\n`
);
is(
record.version,
maxVersionMap[lookupKey(record)],
`Expected record ${record.name} to be version ${
maxVersionMap[lookupKey(record)]
}, but found version ${record.version}\n`
);
}
const expectedSize = Object.keys(maxVersionMap).length;
is(
retrievedRecords.length,
expectedSize,
`Expected retrieved records to be the same size as the name-to-version map (
${expectedSize}
), but found ${retrievedRecords.length}\n`
);
TranslationsParent.unmockTranslationsEngine();
});
|