1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
|
#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Firefox about:memory log parser.
import argparse
import gzip
import json
from collections import defaultdict
# This value comes from nsIMemoryReporter.idl.
KIND_HEAP = 1
def path_total(data, path):
"""
Calculates the sum for the given data point path and its children. If
path does not end with a '/' then only the value for the exact path is
returned.
"""
path_totals = defaultdict(int)
# Bookkeeping for calculating the heap-unclassified measurement.
explicit_heap = defaultdict(int)
heap_allocated = defaultdict(int)
discrete = not path.endswith("/")
def match(value):
"""
Helper that performs either an explicit match or a prefix match
depending on the format of the path passed in.
"""
if discrete:
return value == path
else:
return value.startswith(path)
def update_bookkeeping(report):
"""
Adds the value to the heap total if this an explicit entry that is a
heap measurement and updates the heap allocated value if necessary.
"""
if report["kind"] == KIND_HEAP and report["path"].startswith("explicit/"):
explicit_heap[report["process"]] += report["amount"]
elif report["path"] == "heap-allocated":
heap_allocated[report["process"]] = report["amount"]
def heap_unclassified(process):
"""
Calculates the heap-unclassified value for the given process. This is
simply the difference between all values reported as heap allocated
under the explicit/ tree and the value reported for heap-allocated by
the allocator.
"""
# Memory reports should always include heap-allocated. If it's missing
# just assert.
assert process in heap_allocated
unclassified = heap_allocated[process] - explicit_heap[process]
# Make sure the value is sane. A misbehaving reporter could lead to
# negative values.
# This assertion fails on Beta while running TP6, in the Google Docs process.
# Disable this for now, but only on Beta. See bug 1735556.
# assert unclassified >= 0, "heap-unclassified was negative: %d" % unclassified
return unclassified
needs_bookkeeping = path in ("explicit/", "explicit/heap-unclassified")
# Process all the reports.
for report in data["reports"]:
if needs_bookkeeping:
update_bookkeeping(report)
if match(report["path"]):
path_totals[report["process"]] += report["amount"]
# Handle special processing for explicit and heap-unclassified.
if path == "explicit/":
# If 'explicit/' is requested we need to add the 'explicit/heap-unclassified'
# node that is generated by about:memory.
for k, v in explicit_heap.items():
path_totals[k] += heap_unclassified(k)
elif path == "explicit/heap-unclassified":
# If 'explicit/heap-unclassified' is requested we need to calculate the
# value as it's generated by about:memory, not explicitly reported.
for k, v in explicit_heap.items():
path_totals[k] = heap_unclassified(k)
return path_totals
def calculate_memory_report_values(
memory_report_path, data_point_path, process_names=None
):
"""
Opens the given memory report file and calculates the value for the given
data point.
:param memory_report_path: Path to the memory report file to parse.
:param data_point_path: Path of the data point to calculate in the memory
report, ie: 'explicit/heap-unclassified'.
:param process_name: Name of processes to limit reports to. ie 'Main'
"""
try:
with open(memory_report_path) as f:
data = json.load(f)
except ValueError:
# Check if the file is gzipped.
with gzip.open(memory_report_path, "rb") as f:
data = json.load(f)
totals = path_total(data, data_point_path)
# If a process name is provided, restricted output to processes matching
# that name.
if process_names is not None:
for k in list(totals.keys()):
if not any([process_name in k for process_name in process_names]):
del totals[k]
return totals
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Extract data points from about:memory reports"
)
parser.add_argument("report", action="store", help="Path to a memory report file.")
parser.add_argument(
"prefix",
action="store",
help="Prefix of data point to measure. "
"If the prefix does not end in a '/' "
"then an exact match is made.",
)
parser.add_argument(
"--proc-filter",
action="store",
nargs="*",
default=None,
help="Process name filter. " "If not provided all processes will be included.",
)
parser.add_argument(
"--mebi",
action="store_true",
help="Output values as mebibytes (instead of bytes)" " to match about:memory.",
)
args = parser.parse_args()
totals = calculate_memory_report_values(args.report, args.prefix, args.proc_filter)
sorted_totals = sorted(totals.items(), key=lambda item: (-item[1], item[0]))
for (k, v) in sorted_totals:
if v:
print("{0}\t".format(k)),
print("")
bytes_per_mebibyte = 1024.0 * 1024.0
for (k, v) in sorted_totals:
if v:
if args.mebi:
print("{0:.2f} MiB".format(v / bytes_per_mebibyte)),
else:
print("{0} bytes".format(v)),
print("\t"),
print("")
|
