1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
|
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import
import os
import pathlib
import re
from perfdocs.logger import PerfDocLogger
from perfdocs.utils import read_yaml
from perfdocs.framework_gatherers import (
RaptorGatherer,
MozperftestGatherer,
TalosGatherer,
)
logger = PerfDocLogger()
# TODO: Implement decorator/searcher to find the classes.
frameworks = {
"raptor": RaptorGatherer,
"mozperftest": MozperftestGatherer,
"Talos": TalosGatherer,
}
class Gatherer(object):
"""
Gatherer produces the tree of the perfdoc's entries found
and can obtain manifest-based test lists. Used by the Verifier.
"""
def __init__(self, workspace_dir):
"""
Initialzie the Gatherer.
:param str workspace_dir: Path to the gecko checkout.
"""
self.workspace_dir = workspace_dir
self._perfdocs_tree = []
self._test_list = []
self.framework_gatherers = {}
@property
def perfdocs_tree(self):
"""
Returns the perfdocs_tree, and computes it
if it doesn't exist.
:return dict: The perfdocs tree containing all
framework perfdoc entries. See `fetch_perfdocs_tree`
for information on the data structure.
"""
if self._perfdocs_tree:
return self._perfdocs_tree
else:
self.fetch_perfdocs_tree()
return self._perfdocs_tree
def fetch_perfdocs_tree(self):
"""
Creates the perfdocs tree with the following structure:
[
{
"path": Path to the perfdocs directory.
"yml": Name of the configuration YAML file.
"rst": Name of the RST file.
"static": Name of the static file.
}, ...
]
This method doesn't return anything. The result can be found in
the perfdocs_tree attribute.
"""
exclude_dir = ["tools/lint", ".hg", "testing/perfdocs"]
for path in pathlib.Path(self.workspace_dir).rglob("perfdocs"):
if any(re.search(d, str(path)) for d in exclude_dir):
continue
files = [f for f in os.listdir(path)]
matched = {"path": str(path), "yml": "", "rst": "", "static": []}
for file in files:
# Add the yml/rst/static file to its key if re finds the searched file
if file == "config.yml" or file == "config.yaml":
matched["yml"] = file
elif file == "index.rst":
matched["rst"] = file
elif file.endswith(".rst"):
matched["static"].append(file)
# Append to structdocs if all the searched files were found
if all(val for val in matched.values() if not type(val) == list):
self._perfdocs_tree.append(matched)
logger.log(
"Found {} perfdocs directories in {}".format(
len(self._perfdocs_tree), [d["path"] for d in self._perfdocs_tree]
)
)
def get_test_list(self, sdt_entry):
"""
Use a perfdocs_tree entry to find the test list for
the framework that was found.
:return: A framework info dictionary with fields: {
'yml_path': Path to YAML,
'yml_content': Content of YAML,
'name': Name of framework,
'test_list': Test list found for the framework
}
"""
# If it was computed before, return it
yaml_path = os.path.join(sdt_entry["path"], sdt_entry["yml"])
for entry in self._test_list:
if entry["yml_path"] == yaml_path:
return entry
# Set up framework entry with meta data
yaml_content = read_yaml(yaml_path)
framework = {
"yml_content": yaml_content,
"yml_path": yaml_path,
"name": yaml_content["name"],
"test_list": {},
}
# Get and then store the frameworks tests
self.framework_gatherers[framework["name"]] = frameworks[framework["name"]](
framework["yml_path"], self.workspace_dir
)
if not yaml_content["static-only"]:
framework["test_list"] = self.framework_gatherers[
framework["name"]
].get_test_list()
self._test_list.append(framework)
return framework
|
