1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
|
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import codecs
import json
import os
import re
from collections import deque
import six
import mozpack.path as mozpath
from mozbuild.preprocessor import Preprocessor
from mozpack.chrome.manifest import (
Manifest,
ManifestBinaryComponent,
ManifestChrome,
ManifestInterfaces,
is_manifest,
parse_manifest,
)
from mozpack.errors import errors
class Component(object):
"""
Class that represents a component in a package manifest.
"""
def __init__(self, name, destdir=""):
if name.find(" ") > 0:
errors.fatal('Malformed manifest: space in component name "%s"' % name)
self._name = name
self._destdir = destdir
def __repr__(self):
s = self.name
if self.destdir:
s += ' destdir="%s"' % self.destdir
return s
@property
def name(self):
return self._name
@property
def destdir(self):
return self._destdir
@staticmethod
def _triples(lst):
"""
Split [1, 2, 3, 4, 5, 6, 7] into [(1, 2, 3), (4, 5, 6)].
"""
return zip(*[iter(lst)] * 3)
KEY_VALUE_RE = re.compile(
r"""
\s* # optional whitespace.
([a-zA-Z0-9_]+) # key.
\s*=\s* # optional space around =.
"([^"]*)" # value without surrounding quotes.
(?:\s+|$)
""",
re.VERBOSE,
)
@staticmethod
def _split_options(string):
"""
Split 'key1="value1" key2="value2"' into
{'key1':'value1', 'key2':'value2'}.
Returned keys and values are all strings.
Throws ValueError if the input is malformed.
"""
options = {}
splits = Component.KEY_VALUE_RE.split(string)
if len(splits) % 3 != 1:
# This should never happen -- we expect to always split
# into ['', ('key', 'val', '')*].
raise ValueError("Bad input")
if splits[0]:
raise ValueError("Unrecognized input " + splits[0])
for key, val, no_match in Component._triples(splits[1:]):
if no_match:
raise ValueError("Unrecognized input " + no_match)
options[key] = val
return options
@staticmethod
def _split_component_and_options(string):
"""
Split 'name key1="value1" key2="value2"' into
('name', {'key1':'value1', 'key2':'value2'}).
Returned name, keys and values are all strings.
Raises ValueError if the input is malformed.
"""
splits = string.strip().split(None, 1)
if not splits:
raise ValueError("No component found")
component = splits[0].strip()
if not component:
raise ValueError("No component found")
if not re.match("[a-zA-Z0-9_-]+$", component):
raise ValueError("Bad component name " + component)
options = Component._split_options(splits[1]) if len(splits) > 1 else {}
return component, options
@staticmethod
def from_string(string):
"""
Create a component from a string.
"""
try:
name, options = Component._split_component_and_options(string)
except ValueError as e:
errors.fatal("Malformed manifest: %s" % e)
return
destdir = options.pop("destdir", "")
if options:
errors.fatal(
"Malformed manifest: options %s not recognized" % options.keys()
)
return Component(name, destdir=destdir)
class PackageManifestParser(object):
"""
Class for parsing of a package manifest, after preprocessing.
A package manifest is a list of file paths, with some syntaxic sugar:
[] designates a toplevel component. Example: [xpcom]
- in front of a file specifies it to be removed
* wildcard support
** expands to all files and zero or more directories
; file comment
The parser takes input from the preprocessor line by line, and pushes
parsed information to a sink object.
The add and remove methods of the sink object are called with the
current Component instance and a path.
"""
def __init__(self, sink):
"""
Initialize the package manifest parser with the given sink.
"""
self._component = Component("")
self._sink = sink
def handle_line(self, str):
"""
Handle a line of input and push the parsed information to the sink
object.
"""
# Remove comments.
str = str.strip()
if not str or str.startswith(";"):
return
if str.startswith("[") and str.endswith("]"):
self._component = Component.from_string(str[1:-1])
elif str.startswith("-"):
str = str[1:]
self._sink.remove(self._component, str)
elif "," in str:
errors.fatal("Incompatible syntax")
else:
self._sink.add(self._component, str)
class PreprocessorOutputWrapper(object):
"""
File-like helper to handle the preprocessor output and send it to a parser.
The parser's handle_line method is called in the relevant errors.context.
"""
def __init__(self, preprocessor, parser):
self._parser = parser
self._pp = preprocessor
def write(self, str):
with errors.context(self._pp.context["FILE"], self._pp.context["LINE"]):
self._parser.handle_line(str)
def preprocess(input, parser, defines={}):
"""
Preprocess the file-like input with the given defines, and send the
preprocessed output line by line to the given parser.
"""
pp = Preprocessor()
pp.context.update(defines)
pp.do_filter("substitution")
pp.out = PreprocessorOutputWrapper(pp, parser)
pp.do_include(input)
def preprocess_manifest(sink, manifest, defines={}):
"""
Preprocess the given file-like manifest with the given defines, and push
the parsed information to a sink. See PackageManifestParser documentation
for more details on the sink.
"""
preprocess(manifest, PackageManifestParser(sink), defines)
class CallDeque(deque):
"""
Queue of function calls to make.
"""
def append(self, function, *args):
deque.append(self, (errors.get_context(), function, args))
def execute(self):
while True:
try:
context, function, args = self.popleft()
except IndexError:
return
if context:
with errors.context(context[0], context[1]):
function(*args)
else:
function(*args)
class SimplePackager(object):
"""
Helper used to translate and buffer instructions from the
SimpleManifestSink to a formatter. Formatters expect some information to be
given first that the simple manifest contents can't guarantee before the
end of the input.
"""
def __init__(self, formatter):
self.formatter = formatter
# Queue for formatter.add_interfaces()/add_manifest() calls.
self._queue = CallDeque()
# Queue for formatter.add_manifest() calls for ManifestChrome.
self._chrome_queue = CallDeque()
# Queue for formatter.add() calls.
self._file_queue = CallDeque()
# All paths containing addons. (key is path, value is whether it
# should be packed or unpacked)
self._addons = {}
# All manifest paths imported.
self._manifests = set()
# All manifest paths included from some other manifest.
self._included_manifests = {}
self._closed = False
# Parsing RDF is complex, and would require an external library to do
# properly. Just go with some hackish but probably sufficient regexp
UNPACK_ADDON_RE = re.compile(
r"""(?:
<em:unpack>true</em:unpack>
|em:unpack=(?P<quote>["']?)true(?P=quote)
)""",
re.VERBOSE,
)
def add(self, path, file):
"""
Add the given BaseFile instance with the given path.
"""
assert not self._closed
if is_manifest(path):
self._add_manifest_file(path, file)
elif path.endswith(".xpt"):
self._queue.append(self.formatter.add_interfaces, path, file)
else:
self._file_queue.append(self.formatter.add, path, file)
if mozpath.basename(path) == "install.rdf":
addon = True
install_rdf = six.ensure_text(file.open().read())
if self.UNPACK_ADDON_RE.search(install_rdf):
addon = "unpacked"
self._add_addon(mozpath.dirname(path), addon)
elif mozpath.basename(path) == "manifest.json":
manifest = six.ensure_text(file.open().read())
try:
parsed = json.loads(manifest)
except ValueError:
pass
if isinstance(parsed, dict) and "manifest_version" in parsed:
self._add_addon(mozpath.dirname(path), True)
def _add_addon(self, path, addon_type):
"""
Add the given BaseFile to the collection of addons if a parent
directory is not already in the collection.
"""
if mozpath.basedir(path, self._addons) is not None:
return
for dir in self._addons:
if mozpath.basedir(dir, [path]) is not None:
del self._addons[dir]
break
self._addons[path] = addon_type
def _add_manifest_file(self, path, file):
"""
Add the given BaseFile with manifest file contents with the given path.
"""
self._manifests.add(path)
base = ""
if hasattr(file, "path"):
# Find the directory the given path is relative to.
b = mozpath.normsep(file.path)
if b.endswith("/" + path) or b == path:
base = os.path.normpath(b[: -len(path)])
for e in parse_manifest(base, path, codecs.getreader("utf-8")(file.open())):
# ManifestResources need to be given after ManifestChrome, so just
# put all ManifestChrome in a separate queue to make them first.
if isinstance(e, ManifestChrome):
# e.move(e.base) just returns a clone of the entry.
self._chrome_queue.append(self.formatter.add_manifest, e.move(e.base))
elif not isinstance(e, (Manifest, ManifestInterfaces)):
self._queue.append(self.formatter.add_manifest, e.move(e.base))
# If a binary component is added to an addon, prevent the addon
# from being packed.
if isinstance(e, ManifestBinaryComponent):
addon = mozpath.basedir(e.base, self._addons)
if addon:
self._addons[addon] = "unpacked"
if isinstance(e, Manifest):
if e.flags:
errors.fatal("Flags are not supported on " + '"manifest" entries')
self._included_manifests[e.path] = path
def get_bases(self, addons=True):
"""
Return all paths under which root manifests have been found. Root
manifests are manifests that are included in no other manifest.
`addons` indicates whether to include addon bases as well.
"""
all_bases = set(
mozpath.dirname(m) for m in self._manifests - set(self._included_manifests)
)
if not addons:
all_bases -= set(self._addons)
else:
# If for some reason some detected addon doesn't have a
# non-included manifest.
all_bases |= set(self._addons)
return all_bases
def close(self):
"""
Push all instructions to the formatter.
"""
self._closed = True
bases = self.get_bases()
broken_bases = sorted(
m
for m, includer in six.iteritems(self._included_manifests)
if mozpath.basedir(m, bases) != mozpath.basedir(includer, bases)
)
for m in broken_bases:
errors.fatal(
'"%s" is included from "%s", which is outside "%s"'
% (m, self._included_manifests[m], mozpath.basedir(m, bases))
)
for base in sorted(bases):
self.formatter.add_base(base, self._addons.get(base, False))
self._chrome_queue.execute()
self._queue.execute()
self._file_queue.execute()
class SimpleManifestSink(object):
"""
Parser sink for "simple" package manifests. Simple package manifests use
the format described in the PackageManifestParser documentation, but don't
support file removals, and require manifests, interfaces and chrome data to
be explicitely listed.
Entries starting with bin/ are searched under bin/ in the FileFinder, but
are packaged without the bin/ prefix.
"""
def __init__(self, finder, formatter):
"""
Initialize the SimpleManifestSink. The given FileFinder is used to
get files matching the patterns given in the manifest. The given
formatter does the packaging job.
"""
self._finder = finder
self.packager = SimplePackager(formatter)
self._closed = False
self._manifests = set()
@staticmethod
def normalize_path(path):
"""
Remove any bin/ prefix.
"""
if mozpath.basedir(path, ["bin"]) == "bin":
return mozpath.relpath(path, "bin")
return path
def add(self, component, pattern):
"""
Add files with the given pattern in the given component.
"""
assert not self._closed
added = False
for p, f in self._finder.find(pattern):
added = True
if is_manifest(p):
self._manifests.add(p)
dest = mozpath.join(component.destdir, SimpleManifestSink.normalize_path(p))
self.packager.add(dest, f)
if not added:
errors.error("Missing file(s): %s" % pattern)
def remove(self, component, pattern):
"""
Remove files with the given pattern in the given component.
"""
assert not self._closed
errors.fatal("Removal is unsupported")
def close(self, auto_root_manifest=True):
"""
Add possibly missing bits and push all instructions to the formatter.
"""
if auto_root_manifest:
# Simple package manifests don't contain the root manifests, so
# find and add them.
paths = [mozpath.dirname(m) for m in self._manifests]
path = mozpath.dirname(mozpath.commonprefix(paths))
for p, f in self._finder.find(mozpath.join(path, "chrome.manifest")):
if p not in self._manifests:
self.packager.add(SimpleManifestSink.normalize_path(p), f)
self.packager.close()
|