diff options
Diffstat (limited to 'toolkit/components/glean/build_scripts')
20 files changed, 2953 insertions, 0 deletions
diff --git a/toolkit/components/glean/build_scripts/glean_parser_ext/cpp.py b/toolkit/components/glean/build_scripts/glean_parser_ext/cpp.py new file mode 100644 index 0000000000..003ae4997c --- /dev/null +++ b/toolkit/components/glean/build_scripts/glean_parser_ext/cpp.py @@ -0,0 +1,139 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Outputter to generate C++ code for metrics. +""" + +import json + +import jinja2 +from glean_parser import metrics, util +from util import generate_metric_ids, generate_ping_ids, get_metrics + + +def cpp_datatypes_filter(value): + """ + A Jinja2 filter that renders C++ literals. + + Based on Python's JSONEncoder, but overrides: + - lists to array literals {} + - strings to "value" + """ + + class CppEncoder(json.JSONEncoder): + def iterencode(self, value): + if isinstance(value, list): + yield "{" + first = True + for subvalue in list(value): + if not first: + yield ", " + yield from self.iterencode(subvalue) + first = False + yield "}" + elif isinstance(value, str): + yield '"' + value + '"' + else: + yield from super().iterencode(value) + + return "".join(CppEncoder().iterencode(value)) + + +def type_name(obj): + """ + Returns the C++ type to use for a given metric object. + """ + + if getattr(obj, "labeled", False): + class_name = util.Camelize(obj.type[8:]) # strips "labeled_" off the front. + label_enum = "DynamicLabel" + if obj.labels and len(obj.labels): + label_enum = f"{util.Camelize(obj.name)}Label" + return f"Labeled<impl::{class_name}Metric, {label_enum}>" + generate_enums = getattr(obj, "_generate_enums", []) # Extra Keys? Reasons? + if len(generate_enums): + for name, _ in generate_enums: + if not len(getattr(obj, name)) and isinstance(obj, metrics.Event): + return util.Camelize(obj.type) + "Metric<NoExtraKeys>" + else: + # we always use the `extra` suffix, + # because we only expose the new event API + suffix = "Extra" + return "{}Metric<{}>".format( + util.Camelize(obj.type), util.Camelize(obj.name) + suffix + ) + return util.Camelize(obj.type) + "Metric" + + +def extra_type_name(typ: str) -> str: + """ + Returns the corresponding Rust type for event's extra key types. + """ + + if typ == "boolean": + return "bool" + elif typ == "string": + return "nsCString" + elif typ == "quantity": + return "uint32_t" + else: + return "UNSUPPORTED" + + +def output_cpp(objs, output_fd, options={}): + """ + Given a tree of objects, output C++ code to the file-like object `output_fd`. + + :param objs: A tree of objects (metrics and pings) as returned from + `parser.parse_objects`. + :param output_fd: Writeable file to write the output to. + :param options: options dictionary. + """ + + # Monkeypatch util.snake_case for the templates to use + util.snake_case = lambda value: value.replace(".", "_").replace("-", "_") + # Monkeypatch util.get_jinja2_template to find templates nearby + + def get_local_template(template_name, filters=()): + env = jinja2.Environment( + loader=jinja2.PackageLoader("cpp", "templates"), + trim_blocks=True, + lstrip_blocks=True, + ) + env.filters["camelize"] = util.camelize + env.filters["Camelize"] = util.Camelize + for filter_name, filter_func in filters: + env.filters[filter_name] = filter_func + return env.get_template(template_name) + + util.get_jinja2_template = get_local_template + get_metric_id = generate_metric_ids(objs) + get_ping_id = generate_ping_ids(objs) + + if "pings" in objs: + template_filename = "cpp_pings.jinja2" + if objs.get("tags"): + del objs["tags"] + else: + template_filename = "cpp.jinja2" + objs = get_metrics(objs) + + template = util.get_jinja2_template( + template_filename, + filters=( + ("cpp", cpp_datatypes_filter), + ("snake_case", util.snake_case), + ("type_name", type_name), + ("extra_type_name", extra_type_name), + ("metric_id", get_metric_id), + ("ping_id", get_ping_id), + ("Camelize", util.Camelize), + ), + ) + + output_fd.write(template.render(all_objs=objs)) + output_fd.write("\n") diff --git a/toolkit/components/glean/build_scripts/glean_parser_ext/jog.py b/toolkit/components/glean/build_scripts/glean_parser_ext/jog.py new file mode 100644 index 0000000000..81d92b0f5d --- /dev/null +++ b/toolkit/components/glean/build_scripts/glean_parser_ext/jog.py @@ -0,0 +1,207 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Outputter to generate Rust code for metrics. +""" + +import enum +import json +import sys + +import jinja2 +from glean_parser import util +from glean_parser.metrics import Rate +from util import type_ids_and_categories + +from js import ID_BITS, PING_INDEX_BITS + +RUNTIME_METRIC_BIT = ID_BITS - 1 +RUNTIME_PING_BIT = PING_INDEX_BITS - 1 + +# The list of all args to CommonMetricData. +# No particular order is required, but I have these in common_metric_data.rs +# order just to be organized. +# Note that this is util.common_metric_args + "dynamic_label" +common_metric_data_args = [ + "name", + "category", + "send_in_pings", + "lifetime", + "disabled", + "dynamic_label", +] + +# List of all metric-type-specific args that JOG understands. +known_extra_args = [ + "time_unit", + "memory_unit", + "allowed_extra_keys", + "reason_codes", + "range_min", + "range_max", + "bucket_count", + "histogram_type", + "numerators", + "ordered_labels", +] + +# List of all ping-specific args that JOG undertsands. +known_ping_args = [ + "name", + "include_client_id", + "send_if_empty", + "precise_timestamps", + "reason_codes", +] + + +def ensure_jog_support_for_args(): + """ + glean_parser or the Glean SDK might add new metric/ping args. + To ensure JOG doesn't fall behind in support, + we check the list of JOG-supported args vs glean_parser's. + We fail the build if glean_parser has one or more we haven't seen before. + """ + + unknown_args = set(util.extra_metric_args) - set(known_extra_args) + + unknown_args |= set(util.ping_args) - set(known_ping_args) + + if len(unknown_args): + print(f"Unknown glean_parser args {unknown_args}") + print("JOG must be updated to support the new args") + sys.exit(1) + + +def load_monkeypatches(): + """ + Monkeypatch jinja template loading because we're not glean_parser. + We're glean_parser_ext. + """ + + # Monkeypatch util.get_jinja2_template to find templates nearby + def get_local_template(template_name, filters=()): + env = jinja2.Environment( + loader=jinja2.PackageLoader("rust", "templates"), + trim_blocks=True, + lstrip_blocks=True, + ) + env.filters["camelize"] = util.camelize + env.filters["Camelize"] = util.Camelize + for filter_name, filter_func in filters: + env.filters[filter_name] = filter_func + return env.get_template(template_name) + + util.get_jinja2_template = get_local_template + + +def output_factory(objs, output_fd, options={}): + """ + Given a tree of objects, output Rust code to the file-like object `output_fd`. + Specifically, Rust code that can generate Rust metrics instances. + + :param objs: A tree of objects (metrics and pings) as returned from + `parser.parse_objects`. + :param output_fd: Writeable file to write the output to. + :param options: options dictionary, presently unused. + """ + + ensure_jog_support_for_args() + load_monkeypatches() + + # Get the metric type ids. Must be the same ids generated in js.py + metric_types, categories = type_ids_and_categories(objs) + + template = util.get_jinja2_template( + "jog_factory.jinja2", + filters=(("snake_case", util.snake_case),), + ) + + output_fd.write( + template.render( + all_objs=objs, + common_metric_data_args=common_metric_data_args, + extra_args=util.extra_args, + metric_types=metric_types, + runtime_metric_bit=RUNTIME_METRIC_BIT, + runtime_ping_bit=RUNTIME_PING_BIT, + ID_BITS=ID_BITS, + ) + ) + output_fd.write("\n") + + +def camel_to_snake(s): + assert "_" not in s, "JOG doesn't encode metric typenames with underscores" + return "".join(["_" + c.lower() if c.isupper() else c for c in s]).lstrip("_") + + +def output_file(objs, output_fd, options={}): + """ + Given a tree of objects, output them to the file-like object `output_fd`. + Specifically, in a format that describes all the metrics and pings defined in objs. + + :param objs: A tree of objects (metrics and pings) as returned from + `parser.parse_objects`. + Presently a dictionary with keys of literals "pings" and "tags" + as well as one key per metric category mapped to lists of + pings, tags, and metrics (respecitvely) + :param output_fd: Writeable file to write the output to. + :param options: options dictionary, presently unused. + """ + + ensure_jog_support_for_args() + + jog_data = {"pings": [], "metrics": {}} + + if "tags" in objs: + del objs["tags"] # JOG has no use for tags. + + pings = objs["pings"] + del objs["pings"] + for ping in pings.values(): + ping_arg_list = [] + for arg in known_ping_args: + if hasattr(ping, arg): + ping_arg_list.append(getattr(ping, arg)) + jog_data["pings"].append(ping_arg_list) + + def encode(value): + if isinstance(value, enum.Enum): + return value.name + if isinstance(value, Rate): # `numerators` for an external Denominator metric + args = [] + for arg_name in common_metric_data_args[:-1]: + args.append(getattr(value, arg_name)) + + # These are deserialized as CommonMetricData. + # CMD have a final param JOG never uses: `dynamic_label` + # It's optional, so we should be able to omit it, but we'd need to + # annotate it with #[serde(default)]... so here we add the sixth + # param as None. + args.append(None) + return args + return json.dumps(value) + + for category, metrics in objs.items(): + dict_cat = jog_data["metrics"].setdefault(category, []) + for metric in metrics.values(): + metric_arg_list = [camel_to_snake(metric.__class__.__name__)] + for arg in common_metric_data_args[:-1]: + if arg in ["category"]: + continue # We don't include the category in each metric. + metric_arg_list.append(getattr(metric, arg)) + extra = {} + for arg in known_extra_args: + if hasattr(metric, arg): + extra[arg] = getattr(metric, arg) + if len(extra): + metric_arg_list.append(extra) + dict_cat.append(metric_arg_list) + + # TODO: Measure the speed gain of removing `indent=2` + json.dump(jog_data, output_fd, sort_keys=True, default=encode, indent=2) diff --git a/toolkit/components/glean/build_scripts/glean_parser_ext/js.py b/toolkit/components/glean/build_scripts/glean_parser_ext/js.py new file mode 100644 index 0000000000..aabae636f9 --- /dev/null +++ b/toolkit/components/glean/build_scripts/glean_parser_ext/js.py @@ -0,0 +1,308 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Outputter to generate C++ code for the JavaScript API for metrics. + +The code for the JavaScript API is a bit special in that we only generate C++ code, +string tables and mapping functions. +The rest is handled by the WebIDL and XPIDL implementation +that uses this code to look up metrics by name. +""" + +import jinja2 +from glean_parser import util +from perfecthash import PerfectHash +from string_table import StringTable +from util import generate_metric_ids, generate_ping_ids, get_metrics + +""" +We need to store several bits of information in the Perfect Hash Map Entry: + +1. An index into the string table to check for string equality with a search key + The perfect hash function will give false-positive for non-existent keys, + so we need to verify these ourselves. +2. Type information to instantiate the correct C++ class +3. The metric's actual ID to lookup the underlying instance. +4. Whether the metric is a "submetric" (generated per-label for labeled_* metrics) +5. Whether the metric was registered at runtime + +We have 64 bits to play with, so we dedicate: + +1. 32 bit to the string table offset. More than enough for a large string table (~60M metrics). +2. 5 bit for the type. That allows for 32 metric types. We're not even close to that yet. +3. 25 bit for the metric ID. That allows for 33.5 million metrics. Let's not go there. +4. 1 bit for signifying that this metric is a submetric +5. 1 bit for signifying that this metric was registered at runtime + +These values are interpolated into the template as well, so changing them here +ensures the generated C++ code follows. +If we ever need more bits for a part (e.g. when we add the 33rd metric type), +we figure out if either the string table indices or the range of possible IDs can be reduced +and adjust the constants below. +""" +ENTRY_WIDTH = 64 +INDEX_BITS = 32 +ID_BITS = 27 # Includes ID_SIGNAL_BITS +ID_SIGNAL_BITS = 2 +TYPE_BITS = 5 + +PING_INDEX_BITS = 16 + +# Size of the PHF intermediate table. +# This ensures the algorithm finds empty slots in the buckets +# with the number of metrics we now have in-tree. +# toolkit/components/telemetry uses 1024, some others 512. +# See https://bugzilla.mozilla.org/show_bug.cgi?id=1822477 +PHF_SIZE = 1024 + + +def ping_entry(ping_id, ping_string_index): + """ + The 2 pieces of information of a ping encoded into a single 32-bit integer. + """ + assert ping_id < 2 ** (32 - PING_INDEX_BITS) + assert ping_string_index < 2**PING_INDEX_BITS + return ping_id << PING_INDEX_BITS | ping_string_index + + +def create_entry(metric_id, type_id, idx): + """ + The 3 pieces of information of a metric encoded into a single 64-bit integer. + """ + return metric_id << INDEX_BITS | type_id << (INDEX_BITS + ID_BITS) | idx + + +def metric_identifier(category, metric_name): + """ + The metric's unique identifier, including the category and name + """ + return f"{category}.{util.camelize(metric_name)}" + + +def type_name(obj): + """ + Returns the C++ type to use for a given metric object. + """ + + if getattr(obj, "labeled", False): + return "GleanLabeled" + return "Glean" + util.Camelize(obj.type) + + +def subtype_name(obj): + """ + Returns the subtype name for labeled metrics. + (e.g. 'boolean' for 'labeled_boolean'). + Returns "" for non-labeled metrics. + """ + if getattr(obj, "labeled", False): + type = obj.type[8:] # strips "labeled_" off the front + return "Glean" + util.Camelize(type) + return "" + + +def output_js(objs, output_fd_h, output_fd_cpp, options={}): + """ + Given a tree of objects, output code for the JS API to the file-like object `output_fd`. + + :param objs: A tree of objects (metrics and pings) as returned from + `parser.parse_objects`. + :param output_fd: Writeable file to write the output to. + :param options: options dictionary. + """ + + # Monkeypatch util.get_jinja2_template to find templates nearby + + def get_local_template(template_name, filters=()): + env = jinja2.Environment( + loader=jinja2.PackageLoader("js", "templates"), + trim_blocks=True, + lstrip_blocks=True, + ) + env.filters["Camelize"] = util.Camelize + for filter_name, filter_func in filters: + env.filters[filter_name] = filter_func + return env.get_template(template_name) + + util.get_jinja2_template = get_local_template + + if "pings" in objs: + write_pings( + {"pings": objs["pings"]}, + output_fd_cpp, + "js_pings.jinja2", + output_fd_h, + "js_pings_h.jinja2", + ) + else: + write_metrics( + get_metrics(objs), output_fd_cpp, "js.jinja2", output_fd_h, "js_h.jinja2" + ) + + +def write_metrics(objs, output_fd, template_filename, output_fd_h, template_filename_h): + """ + Given a tree of objects `objs`, output metrics-only code for the JS API to the + file-like object `output_fd` using template `template_filename` + """ + + template = util.get_jinja2_template( + template_filename, + ) + + assert ( + INDEX_BITS + TYPE_BITS + ID_BITS <= ENTRY_WIDTH + ), "INDEX_BITS, TYPE_BITS, or ID_BITS are larger than allowed" + + get_metric_id = generate_metric_ids(objs) + # Mapping from a metric's identifier to the entry (metric ID | type id | index) + metric_id_mapping = {} + categories = [] + + category_string_table = StringTable() + metric_string_table = StringTable() + # Mapping from a type name to its ID + metric_type_ids = {} + + for category_name, objs in get_metrics(objs).items(): + category_camel = util.camelize(category_name) + id = category_string_table.stringIndex(category_camel) + categories.append((category_camel, id)) + + for metric in objs.values(): + identifier = metric_identifier(category_camel, metric.name) + metric_type_tuple = (type_name(metric), subtype_name(metric)) + if metric_type_tuple in metric_type_ids: + type_id, _ = metric_type_ids[metric_type_tuple] + else: + type_id = len(metric_type_ids) + 1 + metric_type_ids[metric_type_tuple] = (type_id, metric.type) + + idx = metric_string_table.stringIndex(identifier) + metric_id = get_metric_id(metric) + entry = create_entry(metric_id, type_id, idx) + metric_id_mapping[identifier] = entry + + # Create a lookup table for the metric categories only + category_string_table = category_string_table.writeToString("gCategoryStringTable") + category_map = [(bytearray(category, "ascii"), id) for (category, id) in categories] + name_phf = PerfectHash(category_map, PHF_SIZE) + category_by_name_lookup = name_phf.cxx_codegen( + name="CategoryByNameLookup", + entry_type="category_entry_t", + lower_entry=lambda x: str(x[1]) + "ul", + key_type="const nsACString&", + key_bytes="aKey.BeginReading()", + key_length="aKey.Length()", + return_type="Maybe<uint32_t>", + return_entry="return category_result_check(aKey, entry);", + ) + + # Create a lookup table for metric's identifiers. + metric_string_table = metric_string_table.writeToString("gMetricStringTable") + metric_map = [ + (bytearray(metric_name, "ascii"), metric_id) + for (metric_name, metric_id) in metric_id_mapping.items() + ] + metric_phf = PerfectHash(metric_map, PHF_SIZE) + metric_by_name_lookup = metric_phf.cxx_codegen( + name="MetricByNameLookup", + entry_type="metric_entry_t", + lower_entry=lambda x: str(x[1]) + "ull", + key_type="const nsACString&", + key_bytes="aKey.BeginReading()", + key_length="aKey.Length()", + return_type="Maybe<uint32_t>", + return_entry="return metric_result_check(aKey, entry);", + ) + + output_fd.write( + template.render( + categories=categories, + metric_id_mapping=metric_id_mapping, + metric_type_ids=metric_type_ids, + entry_width=ENTRY_WIDTH, + index_bits=INDEX_BITS, + id_bits=ID_BITS, + type_bits=TYPE_BITS, + id_signal_bits=ID_SIGNAL_BITS, + category_string_table=category_string_table, + category_by_name_lookup=category_by_name_lookup, + metric_string_table=metric_string_table, + metric_by_name_lookup=metric_by_name_lookup, + ) + ) + output_fd.write("\n") + + output_fd_h.write( + util.get_jinja2_template(template_filename_h).render( + index_bits=INDEX_BITS, + id_bits=ID_BITS, + type_bits=TYPE_BITS, + id_signal_bits=ID_SIGNAL_BITS, + num_categories=len(categories), + num_metrics=len(metric_id_mapping.items()), + ) + ) + output_fd_h.write("\n") + + +def write_pings(objs, output_fd, template_filename, output_fd_h, template_filename_h): + """ + Given a tree of objects `objs`, output pings-only code for the JS API to the + file-like object `output_fd` using template `template_filename` + """ + + template = util.get_jinja2_template( + template_filename, + filters=(), + ) + + ping_string_table = StringTable() + get_ping_id = generate_ping_ids(objs) + # The map of a ping's name to its entry (a combination of a monotonic + # integer and its index in the string table) + pings = {} + for ping_name in objs["pings"].keys(): + ping_id = get_ping_id(ping_name) + ping_camel = util.camelize(ping_name) + pings[ping_camel] = ping_entry( + ping_id, ping_string_table.stringIndex(ping_camel) + ) + + ping_map = [ + (bytearray(ping_name, "ascii"), ping_entry) + for (ping_name, ping_entry) in pings.items() + ] + ping_string_table = ping_string_table.writeToString("gPingStringTable") + ping_phf = PerfectHash(ping_map, PHF_SIZE) + ping_by_name_lookup = ping_phf.cxx_codegen( + name="PingByNameLookup", + entry_type="ping_entry_t", + lower_entry=lambda x: str(x[1]), + key_type="const nsACString&", + key_bytes="aKey.BeginReading()", + key_length="aKey.Length()", + return_type="Maybe<uint32_t>", + return_entry="return ping_result_check(aKey, entry);", + ) + + output_fd.write( + template.render( + ping_index_bits=PING_INDEX_BITS, + ping_by_name_lookup=ping_by_name_lookup, + ping_string_table=ping_string_table, + ) + ) + output_fd.write("\n") + + output_fd_h.write( + util.get_jinja2_template(template_filename_h).render( + num_pings=len(pings.items()), + ) + ) + output_fd_h.write("\n") diff --git a/toolkit/components/glean/build_scripts/glean_parser_ext/run_glean_parser.py b/toolkit/components/glean/build_scripts/glean_parser_ext/run_glean_parser.py new file mode 100644 index 0000000000..1d7d97cf73 --- /dev/null +++ b/toolkit/components/glean/build_scripts/glean_parser_ext/run_glean_parser.py @@ -0,0 +1,234 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import sys +from pathlib import Path + +import cpp +import jinja2 +import jog +import rust +from glean_parser import lint, parser, translate, util +from mozbuild.util import FileAvoidWrite, memoize +from util import generate_metric_ids + +import js + + +@memoize +def get_deps(): + # Any imported python module is added as a dep automatically, + # so we only need the index and the templates. + return { + *[str(p) for p in (Path(os.path.dirname(__file__)) / "templates").iterdir()], + } + + +class ParserError(Exception): + """Thrown from parse if something goes wrong""" + + pass + + +GIFFT_TYPES = { + "Event": ["event"], + "Histogram": ["timing_distribution", "memory_distribution", "custom_distribution"], + "Scalar": [ + "boolean", + "labeled_boolean", + "counter", + "labeled_counter", + "string", + "string_list", + "timespan", + "uuid", + "datetime", + "quantity", + "rate", + "url", + ], +} + + +def get_parser_options(moz_app_version): + app_version_major = moz_app_version.split(".", 1)[0] + return { + "allow_reserved": False, + "expire_by_version": int(app_version_major), + } + + +def parse(args): + """ + Parse and lint the input files, + then return the parsed objects for further processing. + """ + + # Unfortunately, GeneratedFile appends `flags` directly after `inputs` + # instead of listifying either, so we need to pull stuff from a *args. + yaml_array = args[:-1] + moz_app_version = args[-1] + + input_files = [Path(x) for x in yaml_array] + + options = get_parser_options(moz_app_version) + + return parse_with_options(input_files, options) + + +def parse_with_options(input_files, options): + # Derived heavily from glean_parser.translate.translate. + # Adapted to how mozbuild sends us a fd, and to expire on versions not dates. + + all_objs = parser.parse_objects(input_files, options) + if util.report_validation_errors(all_objs): + raise ParserError("found validation errors during parse") + + nits = lint.lint_metrics(all_objs.value, options) + if nits is not None and any(nit.check_name != "EXPIRED" for nit in nits): + # Treat Warnings as Errors in FOG. + # But don't fail the whole build on expired metrics (it blocks testing). + raise ParserError("glinter nits found during parse") + + objects = all_objs.value + + translate.transform_metrics(objects) + + return objects, options + + +def main(cpp_fd, *args): + def open_output(filename): + return FileAvoidWrite(os.path.join(os.path.dirname(cpp_fd.name), filename)) + + [js_h_path, js_cpp_path, rust_path] = args[-3:] + args = args[:-3] + all_objs, options = parse(args) + + cpp.output_cpp(all_objs, cpp_fd, options) + + with open_output(js_h_path) as js_fd: + with open_output(js_cpp_path) as js_cpp_fd: + js.output_js(all_objs, js_fd, js_cpp_fd, options) + + # We only need this info if we're dealing with pings. + ping_names_by_app_id = {} + if "pings" in all_objs: + import sys + from os import path + + from buildconfig import topsrcdir + + sys.path.append(path.join(path.dirname(__file__), path.pardir, path.pardir)) + from metrics_index import pings_by_app_id + + for app_id, ping_yamls in pings_by_app_id.items(): + input_files = [Path(path.join(topsrcdir, x)) for x in ping_yamls] + ping_objs, _ = parse_with_options(input_files, options) + ping_names_by_app_id[app_id] = ping_objs["pings"].keys() + + with open_output(rust_path) as rust_fd: + rust.output_rust(all_objs, rust_fd, ping_names_by_app_id, options) + + return get_deps() + + +def gifft_map(output_fd, *args): + probe_type = args[-1] + args = args[:-1] + all_objs, options = parse(args) + + # Events also need to output maps from event extra enum to strings. + # Sadly we need to generate code for all possible events, not just mirrored. + # Otherwise we won't compile. + if probe_type == "Event": + output_path = Path(os.path.dirname(output_fd.name)) + with FileAvoidWrite(output_path / "EventExtraGIFFTMaps.cpp") as cpp_fd: + output_gifft_map(output_fd, probe_type, all_objs, cpp_fd) + else: + output_gifft_map(output_fd, probe_type, all_objs, None) + + return get_deps() + + +def output_gifft_map(output_fd, probe_type, all_objs, cpp_fd): + get_metric_id = generate_metric_ids(all_objs) + ids_to_probes = {} + for category_name, objs in all_objs.items(): + for metric in objs.values(): + if ( + hasattr(metric, "telemetry_mirror") + and metric.telemetry_mirror is not None + ): + info = (metric.telemetry_mirror, f"{category_name}.{metric.name}") + if metric.type in GIFFT_TYPES[probe_type]: + if any( + metric.telemetry_mirror == value[0] + for value in ids_to_probes.values() + ): + print( + f"Telemetry mirror {metric.telemetry_mirror} already registered", + file=sys.stderr, + ) + sys.exit(1) + ids_to_probes[get_metric_id(metric)] = info + # If we don't support a mirror for this metric type: build error. + elif not any( + [ + metric.type in types_for_probe + for types_for_probe in GIFFT_TYPES.values() + ] + ): + print( + f"Glean metric {category_name}.{metric.name} is of type {metric.type}" + " which can't be mirrored (we don't know how).", + file=sys.stderr, + ) + sys.exit(1) + + env = jinja2.Environment( + loader=jinja2.PackageLoader("run_glean_parser", "templates"), + trim_blocks=True, + lstrip_blocks=True, + ) + env.filters["snake_case"] = lambda value: value.replace(".", "_").replace("-", "_") + env.filters["Camelize"] = util.Camelize + template = env.get_template("gifft.jinja2") + output_fd.write( + template.render( + ids_to_probes=ids_to_probes, + probe_type=probe_type, + id_bits=js.ID_BITS, + id_signal_bits=js.ID_SIGNAL_BITS, + runtime_metric_bit=jog.RUNTIME_METRIC_BIT, + ) + ) + output_fd.write("\n") + + # Events also need to output maps from event extra enum to strings. + # Sadly we need to generate code for all possible events, not just mirrored. + # Otherwise we won't compile. + if probe_type == "Event": + template = env.get_template("gifft_events.jinja2") + cpp_fd.write(template.render(all_objs=all_objs)) + cpp_fd.write("\n") + + +def jog_factory(output_fd, *args): + all_objs, options = parse(args) + jog.output_factory(all_objs, output_fd, options) + return get_deps() + + +def jog_file(output_fd, *args): + all_objs, options = parse(args) + jog.output_file(all_objs, output_fd, options) + return get_deps() + + +if __name__ == "__main__": + main(sys.stdout, *sys.argv[1:]) diff --git a/toolkit/components/glean/build_scripts/glean_parser_ext/rust.py b/toolkit/components/glean/build_scripts/glean_parser_ext/rust.py new file mode 100644 index 0000000000..615784b481 --- /dev/null +++ b/toolkit/components/glean/build_scripts/glean_parser_ext/rust.py @@ -0,0 +1,283 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Outputter to generate Rust code for metrics. +""" + +import enum +import json + +import jinja2 +from glean_parser import util +from glean_parser.metrics import CowString, Event, Rate +from util import generate_metric_ids, generate_ping_ids, get_metrics + +from js import ID_BITS, ID_SIGNAL_BITS + +# The list of all args to CommonMetricData. +# No particular order is required, but I have these in common_metric_data.rs +# order just to be organized. +common_metric_data_args = [ + "name", + "category", + "send_in_pings", + "lifetime", + "disabled", + "dynamic_label", +] + + +def rust_datatypes_filter(value): + """ + A Jinja2 filter that renders Rust literals. + + Based on Python's JSONEncoder, but overrides: + - dicts and sets to raise an error + - sets to vec![] (used in labels) + - enums to become Class::Value + - lists to vec![] (used in send_in_pings) + - null to None + - strings to "value".into() + - Rate objects to a CommonMetricData initializer + (for external Denominators' Numerators lists) + """ + + class RustEncoder(json.JSONEncoder): + def iterencode(self, value): + if isinstance(value, dict): + raise ValueError("RustEncoder doesn't know dicts {}".format(str(value))) + elif isinstance(value, enum.Enum): + yield (value.__class__.__name__ + "::" + util.Camelize(value.name)) + elif isinstance(value, set): + yield from self.iterencode(sorted(list(value))) + elif isinstance(value, list): + yield "vec![" + first = True + for subvalue in list(value): + if not first: + yield ", " + yield from self.iterencode(subvalue) + first = False + yield "]" + elif value is None: + yield "None" + # CowString is also a 'str' but is a special case. + # Ensure its case is handled before str's (below). + elif isinstance(value, CowString): + yield f'::std::borrow::Cow::from("{value.inner}")' + elif isinstance(value, str): + yield '"' + value + '".into()' + elif isinstance(value, Rate): + yield "CommonMetricData {" + for arg_name in common_metric_data_args: + if hasattr(value, arg_name): + yield f"{arg_name}: " + yield from self.iterencode(getattr(value, arg_name)) + yield ", " + yield " ..Default::default()}" + else: + yield from super().iterencode(value) + + return "".join(RustEncoder().iterencode(value)) + + +def ctor(obj): + """ + Returns the scope and name of the constructor to use for a metric object. + Necessary because LabeledMetric<T> is constructed using LabeledMetric::new + not LabeledMetric<T>::new + """ + if getattr(obj, "labeled", False): + return "LabeledMetric::new" + return class_name(obj.type) + "::new" + + +def type_name(obj): + """ + Returns the Rust type to use for a given metric or ping object. + """ + + if getattr(obj, "labeled", False): + label_enum = "super::DynamicLabel" + if obj.labels and len(obj.labels): + label_enum = f"{util.Camelize(obj.name)}Label" + return f"LabeledMetric<Labeled{class_name(obj.type)}, {label_enum}>" + generate_enums = getattr(obj, "_generate_enums", []) # Extra Keys? Reasons? + if len(generate_enums): + for name, _ in generate_enums: + if not len(getattr(obj, name)) and isinstance(obj, Event): + return class_name(obj.type) + "<NoExtraKeys>" + else: + # we always use the `extra` suffix, + # because we only expose the new event API + suffix = "Extra" + return "{}<{}>".format( + class_name(obj.type), util.Camelize(obj.name) + suffix + ) + return class_name(obj.type) + + +def extra_type_name(typ: str) -> str: + """ + Returns the corresponding Rust type for event's extra key types. + """ + + if typ == "boolean": + return "bool" + elif typ == "string": + return "String" + elif typ == "quantity": + return "u32" + else: + return "UNSUPPORTED" + + +def class_name(obj_type): + """ + Returns the Rust class name for a given metric or ping type. + """ + if obj_type == "ping": + return "Ping" + if obj_type.startswith("labeled_"): + obj_type = obj_type[8:] + return util.Camelize(obj_type) + "Metric" + + +def extra_keys(allowed_extra_keys): + """ + Returns the &'static [&'static str] ALLOWED_EXTRA_KEYS for impl ExtraKeys + """ + return "&[" + ", ".join(map(lambda key: '"' + key + '"', allowed_extra_keys)) + "]" + + +def output_rust(objs, output_fd, ping_names_by_app_id, options={}): + """ + Given a tree of objects, output Rust code to the file-like object `output_fd`. + + :param objs: A tree of objects (metrics and pings) as returned from + `parser.parse_objects`. + :param output_fd: Writeable file to write the output to. + :param ping_names_by_app_id: A map of app_ids to lists of ping names. + Used to determine which custom pings to register. + :param options: options dictionary, presently unused. + """ + + # Monkeypatch util.snake_case for the templates to use + util.snake_case = lambda value: value.replace(".", "_").replace("-", "_") + # Monkeypatch util.get_jinja2_template to find templates nearby + + def get_local_template(template_name, filters=()): + env = jinja2.Environment( + loader=jinja2.PackageLoader("rust", "templates"), + trim_blocks=True, + lstrip_blocks=True, + ) + env.filters["camelize"] = util.camelize + env.filters["Camelize"] = util.Camelize + for filter_name, filter_func in filters: + env.filters[filter_name] = filter_func + return env.get_template(template_name) + + util.get_jinja2_template = get_local_template + get_metric_id = generate_metric_ids(objs) + get_ping_id = generate_ping_ids(objs) + + # Map from a tuple (const, typ) to an array of tuples (id, path) + # where: + # const: The Rust constant name to be used for the lookup map + # typ: The metric type to be stored in the lookup map + # id: The numeric metric ID + # path: The fully qualified path to the metric object in Rust + # + # This map is only filled for metrics, not for pings. + # + # Example: + # + # ("COUNTERS", "CounterMetric") -> [(1, "test_only::clicks"), ...] + objs_by_type = {} + + # Map from a metric ID to the fully qualified path of the event object in Rust. + # Required for the special handling of event lookups. + # + # Example: + # + # 17 -> "test_only::an_event" + events_by_id = {} + + # Map from a labeled type (e.g. "counter") to a map from metric ID to the + # fully qualified path of the labeled metric object in Rust paired with + # whether the labeled metric has an enum. + # Required for the special handling of labeled metric lookups. + # + # Example: + # + # "counter" -> 42 -> ("test_only::mabels_kitchen_counters", false) + labeleds_by_id_by_type = {} + + if "pings" in objs: + template_filename = "rust_pings.jinja2" + objs = {"pings": objs["pings"]} + else: + template_filename = "rust.jinja2" + objs = get_metrics(objs) + for category_name, category_value in objs.items(): + for metric in category_value.values(): + # The constant is all uppercase and suffixed by `_MAP` + const_name = util.snake_case(metric.type).upper() + "_MAP" + typ = type_name(metric) + key = (const_name, typ) + + metric_name = util.snake_case(metric.name) + category_snake = util.snake_case(category_name) + full_path = f"{category_snake}::{metric_name}" + + if metric.type == "event": + events_by_id[get_metric_id(metric)] = full_path + continue + + if getattr(metric, "labeled", False): + labeled_type = metric.type[8:] + if labeled_type not in labeleds_by_id_by_type: + labeleds_by_id_by_type[labeled_type] = {} + labeleds_by_id_by_type[labeled_type][get_metric_id(metric)] = ( + full_path, + metric.labels and len(metric.labels), + ) + continue + + if key not in objs_by_type: + objs_by_type[key] = [] + objs_by_type[key].append((get_metric_id(metric), full_path)) + + # Now for the modules for each category. + template = util.get_jinja2_template( + template_filename, + filters=( + ("rust", rust_datatypes_filter), + ("snake_case", util.snake_case), + ("type_name", type_name), + ("extra_type_name", extra_type_name), + ("ctor", ctor), + ("extra_keys", extra_keys), + ("metric_id", get_metric_id), + ("ping_id", get_ping_id), + ), + ) + + output_fd.write( + template.render( + all_objs=objs, + common_metric_data_args=common_metric_data_args, + metric_by_type=objs_by_type, + extra_args=util.extra_args, + events_by_id=events_by_id, + labeleds_by_id_by_type=labeleds_by_id_by_type, + submetric_bit=ID_BITS - ID_SIGNAL_BITS, + ping_names_by_app_id=ping_names_by_app_id, + ) + ) + output_fd.write("\n") diff --git a/toolkit/components/glean/build_scripts/glean_parser_ext/string_table.py b/toolkit/components/glean/build_scripts/glean_parser_ext/string_table.py new file mode 100644 index 0000000000..2cf5cb68e7 --- /dev/null +++ b/toolkit/components/glean/build_scripts/glean_parser_ext/string_table.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from io import StringIO + + +class StringTable: + """Manages a string table and allows C style serialization to a file.""" + + def __init__(self): + self.current_index = 0 + self.table = {} + + def c_strlen(self, string): + """The length of a string including the null terminating character. + :param string: the input string. + """ + return len(string) + 1 + + def stringIndex(self, string): + """Returns the index in the table of the provided string. Adds the string to + the table if it's not there. + :param string: the input string. + """ + if string in self.table: + return self.table[string] + result = self.current_index + self.table[string] = result + self.current_index += self.c_strlen(string) + return result + + def stringIndexes(self, strings): + """Returns a list of indexes for the provided list of strings. + Adds the strings to the table if they are not in it yet. + :param strings: list of strings to put into the table. + """ + return [self.stringIndex(s) for s in strings] + + def writeToString(self, name): + """Writes the string table to a string as a C const char array. + + See `writeDefinition` for details + + :param name: the name of the output array. + """ + + output = StringIO() + self.writeDefinition(output, name) + return output.getvalue() + + def writeDefinition(self, f, name): + """Writes the string table to a file as a C const char array. + + This writes out the string table as one single C char array for memory + size reasons, separating the individual strings with '\0' characters. + This way we can index directly into the string array and avoid the additional + storage costs for the pointers to them (and potential extra relocations for those). + + :param f: the output stream. + :param name: the name of the output array. + """ + entries = self.table.items() + + # Avoid null-in-string warnings with GCC and potentially + # overlong string constants; write everything out the long way. + def explodeToCharArray(string): + def toCChar(s): + if s == "'": + return "'\\''" + return "'%s'" % s + + return ", ".join(map(toCChar, string)) + + f.write("#if defined(_MSC_VER) && !defined(__clang__)\n") + f.write("const char %s[] = {\n" % name) + f.write("#else\n") + f.write("constexpr char %s[] = {\n" % name) + f.write("#endif\n") + for string, offset in sorted(entries, key=lambda x: x[1]): + if "*/" in string: + raise ValueError( + "String in string table contains unexpected sequence '*/': %s" + % string + ) + + e = explodeToCharArray(string) + if e: + f.write( + " /* %5d - \"%s\" */ %s, '\\0',\n" + % (offset, string, explodeToCharArray(string)) + ) + else: + f.write(" /* %5d - \"%s\" */ '\\0',\n" % (offset, string)) + f.write("};\n\n") diff --git a/toolkit/components/glean/build_scripts/glean_parser_ext/templates/cpp.jinja2 b/toolkit/components/glean/build_scripts/glean_parser_ext/templates/cpp.jinja2 new file mode 100644 index 0000000000..2a4e40d6ac --- /dev/null +++ b/toolkit/components/glean/build_scripts/glean_parser_ext/templates/cpp.jinja2 @@ -0,0 +1,101 @@ +// -*- mode: C++ -*- + +// AUTOGENERATED BY glean_parser. DO NOT EDIT. +{# The rendered source is autogenerated, but this +Jinja2 template is not. Please file bugs! #} + +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_Metrics_h +#define mozilla_Metrics_h + +#include "mozilla/glean/bindings/MetricTypes.h" +#include "mozilla/Maybe.h" +#include "nsTArray.h" +#include "nsPrintfCString.h" + +#include <tuple> + +namespace mozilla::glean { + +{%- macro generate_extra_keys(obj) -%} +{% for name, suffix in obj["_generate_enums"] %} +{# we always use the `extra` suffix, because we only expose the new event API #} +{% set suffix = "Extra" %} +{% if obj|attr(name)|length %} + {{ extra_keys_with_types(obj, name, suffix)|indent }} +{% endif %} +{% endfor %} +{%- endmacro -%} + +{%- macro extra_keys_with_types(obj, name, suffix) -%} +struct {{ obj.name|Camelize }}{{ suffix }} { + {% for item, type in obj|attr(name) %} + mozilla::Maybe<{{type|extra_type_name}}> {{ item|camelize }}; + {% endfor %} + + std::tuple<nsTArray<nsCString>, nsTArray<nsCString>> ToFfiExtra() const { + nsTArray<nsCString> extraKeys; + nsTArray<nsCString> extraValues; + {% for item, type in obj|attr(name) %} + if ({{item|camelize}}) { + extraKeys.AppendElement()->AssignASCII("{{item}}"); + {% if type == "string" %} + extraValues.EmplaceBack({{item|camelize}}.value()); + {% elif type == "boolean" %} + extraValues.AppendElement()->AssignASCII({{item|camelize}}.value() ? "true" : "false"); + {% elif type == "quantity" %} + extraValues.EmplaceBack(nsPrintfCString("%d", {{item|camelize}}.value())); + {% else %} +#error "Glean: Invalid extra key type for metric {{obj.category}}.{{obj.name}}, defined in: {{obj.defined_in['filepath']}}:{{obj.defined_in['line']}})" + {% endif %} + } + {% endfor %} + return std::make_tuple(std::move(extraKeys), std::move(extraValues)); + } +}; +{%- endmacro -%} + +{# Okay, so though `enum class` means we can't pollute the namespace with the + enum variants' identifiers, there's no guarantee there isn't some + preprocessor #define lying in wait to collide with us. Using CamelCase + helps, but isn't foolproof (X11/X.h has `#define Success 0`). + So we prefix it. I chose `e` (for `enum`) for the prefix. #} +{%- macro generate_label_enum(obj) -%} +enum class {{ obj.name|Camelize }}Label: uint16_t { + {% for label in obj.ordered_labels %} + e{{ label|Camelize }} = {{loop.index0}}, + {% endfor %} + e__Other__, +}; +{%- endmacro %} + +struct NoExtraKeys; +enum class DynamicLabel: uint16_t { }; + +{% for category_name, objs in all_objs.items() %} +namespace {{ category_name|snake_case }} { + {% for obj in objs.values() %} + /** + * generated from {{ category_name }}.{{ obj.name }} + */ + {% if obj|attr("_generate_enums") %} +{{ generate_extra_keys(obj) }} + {%- endif %} + {% if obj.labeled and obj.labels and obj.labels|length %} + {{ generate_label_enum(obj)|indent }} + {% endif %} + /** + * {{ obj.description|wordwrap() | replace('\n', '\n * ') }} + */ + constexpr impl::{{ obj|type_name }} {{obj.name|snake_case }}({{obj|metric_id}}); + + {% endfor %} +} +{% endfor %} + +} // namespace mozilla::glean + +#endif // mozilla_Metrics_h diff --git a/toolkit/components/glean/build_scripts/glean_parser_ext/templates/cpp_pings.jinja2 b/toolkit/components/glean/build_scripts/glean_parser_ext/templates/cpp_pings.jinja2 new file mode 100644 index 0000000000..f877cb9685 --- /dev/null +++ b/toolkit/components/glean/build_scripts/glean_parser_ext/templates/cpp_pings.jinja2 @@ -0,0 +1,30 @@ +// -*- mode: C++ -*- + +// AUTOGENERATED BY glean_parser. DO NOT EDIT. +{# The rendered source is autogenerated, but this +Jinja2 template is not. Please file bugs! #} + +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_glean_Pings_h +#define mozilla_glean_Pings_h + +#include "mozilla/glean/bindings/Ping.h" + +namespace mozilla::glean_pings { + +{% for obj in all_objs['pings'].values() %} +/* + * Generated from {{ obj.name }}. + * + * {{ obj.description|wordwrap() | replace('\n', '\n * ') }} + */ +constexpr glean::impl::Ping {{ obj.name|Camelize }}({{ obj.name|ping_id }}); + +{% endfor %} + +} // namespace mozilla::glean_pings + +#endif // mozilla_glean_Pings_h diff --git a/toolkit/components/glean/build_scripts/glean_parser_ext/templates/gifft.jinja2 b/toolkit/components/glean/build_scripts/glean_parser_ext/templates/gifft.jinja2 new file mode 100644 index 0000000000..4cfb5f242d --- /dev/null +++ b/toolkit/components/glean/build_scripts/glean_parser_ext/templates/gifft.jinja2 @@ -0,0 +1,75 @@ +// -*- mode: C++ -*- + +/* This file is auto-generated by run_glean_parser.py. + It is only for internal use by types in + toolkit/components/glean/bindings/private */ +{# The rendered source is autogenerated, but this +Jinja2 template is not. Please file bugs! #} + +#include "mozilla/AppShutdown.h" +#include "mozilla/ClearOnShutdown.h" +#include "mozilla/glean/bindings/GleanJSMetricsLookup.h" +#include "mozilla/glean/bindings/jog/JOG.h" +#include "mozilla/Maybe.h" +#include "mozilla/Telemetry.h" +#include "nsIThread.h" +#include "nsThreadUtils.h" + +#ifndef mozilla_glean_{{ probe_type }}GifftMap_h +#define mozilla_glean_{{ probe_type }}GifftMap_h + +#define DYNAMIC_METRIC_BIT ({{runtime_metric_bit}}) +#define GLEAN_METRIC_ID(id) ((id) & ((1ULL << {{id_bits}}) - 1)) + +namespace mozilla::glean { + +using Telemetry::{{ probe_type }}ID; + +{% if probe_type == "Scalar" %} +static inline bool IsSubmetricId(uint32_t aId) { + // Submetrics have the 2^{{id_bits - id_signal_bits}} bit set. + // (ID_BITS - ID_SIGNAL_BITS, keep it in sync with js.py). + return (aId & (1 << {{id_bits - id_signal_bits}})) > 0; +} +{% endif %} + +static{% if probe_type == "Event" or probe_type == "Scalar" %} inline{% endif %} Maybe<{{ probe_type }}ID> {{ probe_type }}IdForMetric(uint32_t aId) { + switch(aId) { +{% for id, (mirror, metric_name) in ids_to_probes.items() %} + case {{ id }}: { // {{ metric_name }} + return Some({{ probe_type }}ID::{{ mirror }}); + } +{% endfor %} + default: { + if (MOZ_UNLIKELY(aId & (1 << DYNAMIC_METRIC_BIT))) { + // Dynamic (runtime-registered) metric. Use its static (compiletime- + // registered) metric's telemetry_mirror mapping. + // ...if applicable. + + // Only JS can use dynamic (runtime-registered) metric ids. + MOZ_ASSERT(NS_IsMainThread()); + + auto metricName = JOG::GetMetricName(aId); + // All of these should have names, but the storage only lasts until + // XPCOMWillShutdown, so it might return `Nothing()`. + if (metricName.isSome()) { + auto maybeMetric = MetricByNameLookup(metricName.ref()); + if (maybeMetric.isSome()) { + uint32_t staticId = GLEAN_METRIC_ID(maybeMetric.value()); + // Let's ensure we don't infinite loop, huh. + MOZ_ASSERT(!(staticId & (1 << DYNAMIC_METRIC_BIT))); + return {{ probe_type }}IdForMetric(staticId); + } + } + } + return Nothing(); + } + } +} + +} // namespace mozilla::glean + +#undef GLEAN_METRIC_ID +#undef DYNAMIC_METRIC_BIT + +#endif // mozilla_glean_{{ probe_type }}GifftMaps_h diff --git a/toolkit/components/glean/build_scripts/glean_parser_ext/templates/gifft_events.jinja2 b/toolkit/components/glean/build_scripts/glean_parser_ext/templates/gifft_events.jinja2 new file mode 100644 index 0000000000..f32640fc19 --- /dev/null +++ b/toolkit/components/glean/build_scripts/glean_parser_ext/templates/gifft_events.jinja2 @@ -0,0 +1,52 @@ +// -*- mode: C++ -*- + +/* This file is auto-generated by run_glean_parser.py. + It is only for internal use by types in + toolkit/components/glean/bindings/private */ +{# The rendered source is autogenerated, but this +Jinja2 template is not. Pleas file bugs! #} + +#include "mozilla/glean/bindings/Event.h" +#include "mozilla/glean/GleanMetrics.h" + +namespace mozilla::glean { + +template <> +/*static*/ const nsCString impl::EventMetric<NoExtraKeys>::ExtraStringForKey(uint32_t aKey) { + MOZ_ASSERT_UNREACHABLE("What are you doing here? No extra keys!"); + return ""_ns; +} + +{% for category_name, objs in all_objs.items() %} +{% for obj in objs.values() %} +{% if obj|attr("_generate_enums") %} +{# we always use the `extra` suffix, because we only expose the new event API #} +{% set suffix = "Extra" %} +{% for name, _ in obj["_generate_enums"] %} +{% if obj|attr(name)|length %} +{% set ns %}{{ category_name|snake_case }}{% endset %} +{% set type %}{{ obj.name|Camelize }}{{ suffix }}{% endset %} +template <> +/*static*/ const nsCString impl::EventMetric<{{ ns }}::{{ type }}>::ExtraStringForKey(uint32_t aKey) { + using {{ ns }}::{{ type }}; + switch (aKey) { +{% if obj|attr("telemetry_mirror") %}{# Optimization: Do not generate switch if not mirrored #} +{% for key, _ in obj|attr(name) %} + case {{loop.index-1}}: { + return "{{ key }}"_ns; + } +{% endfor %} +{% endif %} + default: { + MOZ_ASSERT_UNREACHABLE("Impossible event key reached."); + return ""_ns; + } + } +} + +{% endif %} +{% endfor %} +{% endif %} +{% endfor %} +{% endfor %} +}; // namespace mozilla::glean diff --git a/toolkit/components/glean/build_scripts/glean_parser_ext/templates/jog_factory.jinja2 b/toolkit/components/glean/build_scripts/glean_parser_ext/templates/jog_factory.jinja2 new file mode 100644 index 0000000000..4b7838a47d --- /dev/null +++ b/toolkit/components/glean/build_scripts/glean_parser_ext/templates/jog_factory.jinja2 @@ -0,0 +1,149 @@ +// -*- mode: Rust -*- + +// AUTOGENERATED BY glean_parser. DO NOT EDIT. +{# The rendered source is autogenerated, but this +Jinja2 template is not. Please file bugs! #} + +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/// This file contains factory implementation information for the +/// JOG Runtime Registration module. +/// It is responsible for being able to build metrics and pings described at runtime. +/// It is generated to keep it in sync with how the runtime definitions are defined. + +use std::borrow::Cow; +use std::sync::atomic::{AtomicU32, Ordering}; +use crate::private::{ + CommonMetricData, + Lifetime, + MemoryUnit, + TimeUnit, + Ping, + LabeledMetric, +{% for metric_type_name in metric_types.keys() if not metric_type_name.startswith('labeled_') %} + {{ metric_type_name|Camelize }}Metric, +{% endfor %}}; +use crate::private::traits::HistogramType; + +pub(crate) static DYNAMIC_METRIC_BIT: u32 = {{runtime_metric_bit}}; +// 2**DYNAMIC_METRIC_BIT + 1 (+1 because we reserve the 0 metric id) +static NEXT_METRIC_ID: AtomicU32 = AtomicU32::new({{2**runtime_metric_bit + 1}}); +#[cfg(feature = "with_gecko")] // only used in submit_ping_by_id, which is gecko-only. +pub(crate) static DYNAMIC_PING_BIT: u32 = {{runtime_ping_bit}}; +// 2**DYNAMIC_PING_BIT + 1 (+1 because we reserve the 0 ping id) +static NEXT_PING_ID: AtomicU32 = AtomicU32::new({{2**runtime_ping_bit + 1}}); + +pub(crate) mod __jog_metric_maps { + use crate::metrics::DynamicLabel; + use crate::private::MetricId; + use crate::private::{ + Ping, + LabeledMetric, + NoExtraKeys, + {% for metric_type_name in metric_types.keys() %} + {{ metric_type_name|Camelize }}Metric, + {% endfor %} + }; + use once_cell::sync::Lazy; + use std::collections::HashMap; + use std::sync::{Arc, RwLock}; + +{% for metric_type_name in metric_types.keys() if metric_type_name != "event" and not metric_type_name.startswith('labeled_') %} + pub static {{ metric_type_name.upper() }}_MAP: Lazy<Arc<RwLock<HashMap<MetricId, {{ metric_type_name|Camelize }}Metric>>>> = + Lazy::new(|| Arc::new(RwLock::new(HashMap::new()))); + +{% endfor %} +{# Labeled metrics are special because they're LabeledMetric<Labeled{Counter|Boolean|...}Metric> #} +{% for metric_type_name in metric_types.keys() if metric_type_name.startswith('labeled_') %} + pub static {{ metric_type_name.upper() }}_MAP: Lazy<Arc<RwLock<HashMap<MetricId, LabeledMetric<{{ metric_type_name|Camelize }}Metric, DynamicLabel>>>>> = + Lazy::new(|| Arc::new(RwLock::new(HashMap::new()))); + +{% endfor %} + pub static PING_MAP: Lazy<Arc<RwLock<HashMap<u32, Ping>>>> = + Lazy::new(|| Arc::new(RwLock::new(HashMap::new()))); + +{# Event metrics are special because they're EventMetric<K> #} + pub static EVENT_MAP: Lazy<Arc<RwLock<HashMap<MetricId, EventMetric<NoExtraKeys>>>>> = + Lazy::new(|| Arc::new(RwLock::new(HashMap::new()))); +} + +#[derive(Debug)] +struct MetricTypeNotFoundError(String); +impl std::error::Error for MetricTypeNotFoundError {} +impl std::fmt::Display for MetricTypeNotFoundError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "Metric type {} not found", self.0) + } +} + +/// Creates and registers a metric, returning its type+id. +pub fn create_and_register_metric( + metric_type: &str, +{# The rest of these are handrolled because it proved easier than maintaining a +map of argument name to argument type. I may regret this if I need it again. #} +{# In order of util.common_metric_args and util.extra_metric_args, because why not. #} + category: String, + name: String, + send_in_pings: Vec<String>, + lifetime: Lifetime, + disabled: bool, + time_unit: Option<TimeUnit>, + memory_unit: Option<MemoryUnit>, + allowed_extra_keys: Option<Vec<String>>, +{# Skipping reason_codes since that's a ping thing. #} + range_min: Option<u64>, + range_max: Option<u64>, + bucket_count: Option<u64>, + histogram_type: Option<HistogramType>, + numerators: Option<Vec<CommonMetricData>>, +{# And, don't forget the list of acceptable labels for a labeled metric. #} + labels: Option<Vec<Cow<'static, str>>>, +) -> Result<(u32, u32), Box<dyn std::error::Error>> { + let metric_id = NEXT_METRIC_ID.fetch_add(1, Ordering::SeqCst); + let metric32 = match metric_type { +{% for metric_type_name, metric_type in metric_types.items() %} + "{{ metric_type_name }}" => { + let metric = {{ metric_type_name|Camelize if not metric_type_name.startswith('labeled_') else "Labeled"}}Metric::{% if metric_type_name == 'event' %}with_runtime_extra_keys{% else %}new{% endif %}(metric_id.into(), CommonMetricData { + {% for arg_name in common_metric_data_args if arg_name in metric_type.args %} + {{ arg_name }}, + {% endfor %} + ..Default::default() + } + {%- for arg_name in metric_type.args if arg_name not in common_metric_data_args -%} + , {{ arg_name }}.unwrap() + {%- endfor -%} + {%- if metric_type_name.startswith('labeled_') -%} + , labels + {%- endif -%} + ); + let metric32: u32 = ({{metric_type.id}} << {{ID_BITS}}) | metric_id; + assert!( + __jog_metric_maps::{{metric_type_name.upper()}}_MAP.write()?.insert(metric_id.into(), metric).is_none(), + "We should never insert a runtime metric with an already-used id." + ); + metric32 + } +{% endfor %} + _ => return Err(Box::new(MetricTypeNotFoundError(metric_type.to_string()))) + }; + Ok((metric32, metric_id)) +} + +/// Creates and registers a ping, returning its id. +pub fn create_and_register_ping( + ping_name: String, + include_client_id: bool, + send_if_empty: bool, + precise_timestamps: bool, + reason_codes: Vec<String>, +) -> Result<u32, Box<dyn std::error::Error>> { + let ping_id = NEXT_PING_ID.fetch_add(1, Ordering::SeqCst); + let ping = Ping::new(ping_name, include_client_id, send_if_empty, precise_timestamps, reason_codes); + assert!( + __jog_metric_maps::PING_MAP.write()?.insert(ping_id.into(), ping).is_none(), + "We should never insert a runtime ping with an already-used id." + ); + Ok(ping_id) +} diff --git a/toolkit/components/glean/build_scripts/glean_parser_ext/templates/js.jinja2 b/toolkit/components/glean/build_scripts/glean_parser_ext/templates/js.jinja2 new file mode 100644 index 0000000000..aa0d741083 --- /dev/null +++ b/toolkit/components/glean/build_scripts/glean_parser_ext/templates/js.jinja2 @@ -0,0 +1,170 @@ +// -*- mode: C++ -*- + +// AUTOGENERATED BY glean_parser. DO NOT EDIT. +{# The rendered source is autogenerated, but this +Jinja2 template is not. Please file bugs! #} + +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/glean/bindings/GleanJSMetricsLookup.h" + +#include "mozilla/PerfectHash.h" +#include "mozilla/Maybe.h" +#include "mozilla/glean/bindings/MetricTypes.h" +#include "mozilla/glean/fog_ffi_generated.h" +#include "nsString.h" + +#define GLEAN_INDEX_BITS ({{index_bits}}) +#define GLEAN_TYPE_BITS ({{type_bits}}) +#define GLEAN_ID_BITS ({{id_bits}}) +#define GLEAN_TYPE_ID(id) ((id) >> GLEAN_ID_BITS) +#define GLEAN_METRIC_ID(id) ((id) & ((1ULL << GLEAN_ID_BITS) - 1)) +#define GLEAN_OFFSET(entry) (entry & ((1ULL << GLEAN_INDEX_BITS) - 1)) + +namespace mozilla::glean { + +// The category lookup table's entry type +using category_entry_t = uint32_t; +// The metric lookup table's entry type +// This is a bitpacked type with {{index_bits}} bits available to index into +// the string table, {{type_bits}} bits available to signify the metric type, +// and the remaining {{id_bits}} bits devoted to {{id_signal_bits}} "signal" +// bits to signify important characteristics (metric's a labeled metric's +// submetric, metric's been registered at runtime) and {{id_bits - id_signal_bits}} bits +// for built-in metric ids. +// Gives room for {{2 ** (id_bits - id_signal_bits)}} of each combination of +// characteristics (which hopefully will prove to be enough). +using metric_entry_t = uint64_t; + +static_assert(GLEAN_INDEX_BITS + GLEAN_TYPE_BITS + GLEAN_ID_BITS == sizeof(metric_entry_t) * 8, "Index, Type, and ID bits need to fit into a metric_entry_t"); +static_assert(GLEAN_TYPE_BITS + GLEAN_ID_BITS <= sizeof(uint32_t) * 8, "Metric Types and IDs need to fit into at most 32 bits"); +static_assert({{ categories|length }} < UINT32_MAX, "Too many metric categories generated."); +static_assert({{ metric_id_mapping|length }} < {{2 ** (id_bits - id_signal_bits)}}, "Too many metrics generated. Need room for {{id_signal_bits}} signal bits."); +static_assert({{ metric_type_ids|length }} < {{2 ** type_bits}}, "Too many different metric types."); + +already_AddRefed<GleanMetric> NewMetricFromId(uint32_t id, nsISupports* aParent) { + uint32_t typeId = GLEAN_TYPE_ID(id); + uint32_t metricId = GLEAN_METRIC_ID(id); + + switch (typeId) { + {% for (type_name, subtype_name), (type_id, original_type) in metric_type_ids.items() %} + case {{ type_id }}: /* {{ original_type }} */ + { + return MakeAndAddRef<{{type_name}}>(metricId{% if subtype_name|length > 0 %}, {{ type_id }}{% endif %}, aParent); + } + {% endfor %} + default: + MOZ_ASSERT_UNREACHABLE("Invalid type ID reached when trying to instantiate a new metric"); + return nullptr; + } +} + +/** + * Create a submetric instance for a labeled metric of the provided type and id for the given label. + * Assigns or retrieves an id for the submetric from the SDK. + * + * @param aParentTypeId - The type of the parent labeled metric identified as a number generated during codegen. + * Only used to identify which X of LabeledX you are so that X can be created here. + * @param aParentMetricId - The metric id for the parent labeled metric. + * @param aLabel - The label for the submetric. Might not adhere to the SDK label format. + * @param aSubmetricId - an outparam which is assigned the submetric's SDK-generated submetric id. + * Used only by GIFFT. + */ +already_AddRefed<GleanMetric> NewSubMetricFromIds(uint32_t aParentTypeId, + uint32_t aParentMetricId, + const nsACString& aLabel, + uint32_t* aSubmetricId, + nsISupports* aParent) { + switch (aParentTypeId) { + {% for (type_name, subtype_name), (type_id, original_type) in metric_type_ids.items() %} + {# TODO: Remove the subtype inclusion clause when we suport the rest of labeled_* #} + {% if subtype_name|length > 0 and original_type in ['labeled_boolean', 'labeled_counter', 'labeled_string'] %} + case {{ type_id }}: { /* {{ original_type }} */ + auto id = impl::fog_{{original_type}}_get(aParentMetricId, &aLabel); + *aSubmetricId = id; + return MakeAndAddRef<{{subtype_name}}>(id, aParent); + } + {% endif %} + {% endfor %} + default: { + MOZ_ASSERT_UNREACHABLE("Invalid type ID for submetric."); + return nullptr; + } + } +} + +static Maybe<uint32_t> category_result_check(const nsACString& aKey, category_entry_t entry); +static Maybe<uint32_t> metric_result_check(const nsACString& aKey, metric_entry_t entry); + +{{ category_string_table }} +static_assert(sizeof(gCategoryStringTable) < UINT32_MAX, "Category string table is too large."); + +{{ category_by_name_lookup }} + +{{ metric_string_table }} +static_assert(sizeof(gMetricStringTable) < {{2 ** index_bits}}, "Metric string table is too large."); + +{{ metric_by_name_lookup }} + +/** + * Get a category's name from the string table. + */ +const char* GetCategoryName(category_entry_t entry) { + MOZ_ASSERT(entry < sizeof(gCategoryStringTable), "Entry identifier offset larger than string table"); + return &gCategoryStringTable[entry]; +} + +/** + * Get a metric's identifier from the string table. + */ +const char* GetMetricIdentifier(metric_entry_t entry) { + uint32_t offset = GLEAN_OFFSET(entry); + MOZ_ASSERT(offset < sizeof(gMetricStringTable), "Entry identifier offset larger than string table"); + return &gMetricStringTable[offset]; +} + +/** + * Check that the found entry is pointing to the right key + * and return it. + * Or return `Nothing()` if the entry was not found. + */ +static Maybe<uint32_t> category_result_check(const nsACString& aKey, category_entry_t entry) { + if (MOZ_UNLIKELY(entry > sizeof(gCategoryStringTable))) { + return Nothing(); + } + if (aKey.EqualsASCII(gCategoryStringTable + entry)) { + return Some(entry); + } + return Nothing(); +} + +/** + * Check if the found entry index is pointing to the right key + * and return the corresponding metric ID. + * Or return `Nothing()` if the entry was not found. + */ +static Maybe<uint32_t> metric_result_check(const nsACString& aKey, uint64_t entry) { + uint32_t metricId = entry >> GLEAN_INDEX_BITS; + uint32_t offset = GLEAN_OFFSET(entry); + + if (offset > sizeof(gMetricStringTable)) { + return Nothing(); + } + + if (aKey.EqualsASCII(gMetricStringTable + offset)) { + return Some(metricId); + } + + return Nothing(); +} + + +#undef GLEAN_INDEX_BITS +#undef GLEAN_ID_BITS +#undef GLEAN_TYPE_ID +#undef GLEAN_METRIC_ID +#undef GLEAN_OFFSET + +} // namespace mozilla::glean diff --git a/toolkit/components/glean/build_scripts/glean_parser_ext/templates/js_h.jinja2 b/toolkit/components/glean/build_scripts/glean_parser_ext/templates/js_h.jinja2 new file mode 100644 index 0000000000..1de790be10 --- /dev/null +++ b/toolkit/components/glean/build_scripts/glean_parser_ext/templates/js_h.jinja2 @@ -0,0 +1,77 @@ +// -*- mode: C++ -*- + +// AUTOGENERATED BY glean_parser. DO NOT EDIT. +{# The rendered source is autogenerated, but this +Jinja2 template is not. Please file bugs! #} + +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_GleanJSMetricsLookup_h +#define mozilla_GleanJSMetricsLookup_h + +#include <cstdint> + +#include "mozilla/AlreadyAddRefed.h" +#include "mozilla/Maybe.h" +#include "mozilla/glean/bindings/GleanMetric.h" +#include "nsStringFwd.h" + +class nsISupports; + +namespace mozilla::glean { + +// The category lookup table's entry type +using category_entry_t = uint32_t; +// The metric lookup table's entry type +// This is a bitpacked type with {{index_bits}} bits available to index into +// the string table, {{type_bits}} bits available to signify the metric type, +// and the remaining {{id_bits}} bits devoted to {{id_signal_bits}} "signal" +// bits to signify important characteristics (metric's a labeled metric's +// submetric, metric's been registered at runtime) and {{id_bits - id_signal_bits}} bits +// for built-in metric ids. +// Gives room for {{2 ** (id_bits - id_signal_bits)}} of each combination of +// characteristics (which hopefully will prove to be enough). +using metric_entry_t = uint64_t; + +already_AddRefed<GleanMetric> NewMetricFromId(uint32_t id, nsISupports* aParent); + +/** + * Create a submetric instance for a labeled metric of the provided type and id for the given label. + * Assigns or retrieves an id for the submetric from the SDK. + * + * @param aParentTypeId - The type of the parent labeled metric identified as a number generated during codegen. + * Only used to identify which X of LabeledX you are so that X can be created here. + * @param aParentMetricId - The metric id for the parent labeled metric. + * @param aLabel - The label for the submetric. Might not adhere to the SDK label format. + * @param aSubmetricId - an outparam which is assigned the submetric's SDK-generated submetric id. + * Used only by GIFFT. + */ +already_AddRefed<GleanMetric> NewSubMetricFromIds(uint32_t aParentTypeId, uint32_t aParentMetricId, const nsACString& aLabel, uint32_t* aSubmetricId, nsISupports* aParent); + +/** + * Get a category's name from the string table. + */ +const char* GetCategoryName(category_entry_t entry); + +/** + * Get a metric's identifier from the string table. + */ +const char* GetMetricIdentifier(metric_entry_t entry); + +/** + * Get a metric's id given its name. + */ +Maybe<uint32_t> MetricByNameLookup(const nsACString&); + +/** + * Get a category's id given its name. + */ +Maybe<uint32_t> CategoryByNameLookup(const nsACString&); + +extern const category_entry_t sCategoryByNameLookupEntries[{{num_categories}}]; +extern const metric_entry_t sMetricByNameLookupEntries[{{num_metrics}}]; + +} // namespace mozilla::glean +#endif // mozilla_GleanJSMetricsLookup_h diff --git a/toolkit/components/glean/build_scripts/glean_parser_ext/templates/js_pings.jinja2 b/toolkit/components/glean/build_scripts/glean_parser_ext/templates/js_pings.jinja2 new file mode 100644 index 0000000000..4fa43832a6 --- /dev/null +++ b/toolkit/components/glean/build_scripts/glean_parser_ext/templates/js_pings.jinja2 @@ -0,0 +1,67 @@ +// -*- mode: C++ -*- + +// AUTOGENERATED BY glean_parser. DO NOT EDIT. +{# The rendered source is autogenerated, but this +Jinja2 template is not. Please file bugs! #} + +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/glean/bindings/GleanJSPingsLookup.h" + +#include "mozilla/PerfectHash.h" +#include "nsString.h" + +#include "mozilla/PerfectHash.h" + +#define GLEAN_PING_INDEX_BITS ({{ping_index_bits}}) +#define GLEAN_PING_ID(entry) ((entry) >> GLEAN_PING_INDEX_BITS) +#define GLEAN_PING_INDEX(entry) ((entry) & ((1UL << GLEAN_PING_INDEX_BITS) - 1)) + +namespace mozilla::glean { + +// Contains the ping id and the index into the ping string table. +using ping_entry_t = uint32_t; + +Maybe<uint32_t> ping_result_check(const nsACString& aKey, ping_entry_t aEntry); + +{{ ping_string_table }} + +{{ ping_by_name_lookup }} + +/** + * Get a ping's name given its entry from the PHF. + */ +const char* GetPingName(ping_entry_t aEntry) { + uint32_t idx = GLEAN_PING_INDEX(aEntry); + MOZ_ASSERT(idx < sizeof(gPingStringTable), "Ping index larger than string table"); + return &gPingStringTable[idx]; +} + +/** + * Check if the found entry is pointing at the correct ping. + * PHF can false-positive a result when the key isn't present, so we check + * for a string match. If it fails, return Nothing(). If we found it, + * return the ping's id. + */ +Maybe<uint32_t> ping_result_check(const nsACString& aKey, ping_entry_t aEntry) { + uint32_t idx = GLEAN_PING_INDEX(aEntry); + uint32_t id = GLEAN_PING_ID(aEntry); + + if (MOZ_UNLIKELY(idx > sizeof(gPingStringTable))) { + return Nothing(); + } + + if (aKey.EqualsASCII(&gPingStringTable[idx])) { + return Some(id); + } + + return Nothing(); +} + +#undef GLEAN_PING_INDEX_BITS +#undef GLEAN_PING_ID +#undef GLEAN_PING_INDEX + +} // namespace mozilla::glean diff --git a/toolkit/components/glean/build_scripts/glean_parser_ext/templates/js_pings_h.jinja2 b/toolkit/components/glean/build_scripts/glean_parser_ext/templates/js_pings_h.jinja2 new file mode 100644 index 0000000000..3052b8549b --- /dev/null +++ b/toolkit/components/glean/build_scripts/glean_parser_ext/templates/js_pings_h.jinja2 @@ -0,0 +1,35 @@ +// -*- mode: C++ -*- + +// AUTOGENERATED BY glean_parser. DO NOT EDIT. +{# The rendered source is autogenerated, but this +Jinja2 template is not. Please file bugs! #} + +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_GleanJSPingsLookup_h +#define mozilla_GleanJSPingsLookup_h + +#include <cstdint> +#include "mozilla/Maybe.h" +#include "nsStringFwd.h" + +namespace mozilla::glean { + +// Contains the ping id and the index into the ping string table. +using ping_entry_t = uint32_t; + +/** + * Get a ping's name given its entry in the PHF. + */ +const char* GetPingName(ping_entry_t aEntry); + +/** + * Get a ping's id given its name. + */ +Maybe<uint32_t> PingByNameLookup(const nsACString&); + +extern const ping_entry_t sPingByNameLookupEntries[{{num_pings}}]; +} // namespace mozilla::glean +#endif // mozilla_GleanJSPingsLookup_h diff --git a/toolkit/components/glean/build_scripts/glean_parser_ext/templates/rust.jinja2 b/toolkit/components/glean/build_scripts/glean_parser_ext/templates/rust.jinja2 new file mode 100644 index 0000000000..cc29805099 --- /dev/null +++ b/toolkit/components/glean/build_scripts/glean_parser_ext/templates/rust.jinja2 @@ -0,0 +1,355 @@ +// -*- mode: Rust -*- + +// AUTOGENERATED BY glean_parser. DO NOT EDIT. +{# The rendered source is autogenerated, but this +Jinja2 template is not. Please file bugs! #} + +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +{% macro generate_extra_keys(obj) -%} +{% for name, _ in obj["_generate_enums"] %} +{# we always use the `extra` suffix, because we only expose the new event API #} +{% set suffix = "Extra" %} +{% if obj|attr(name)|length %} + {{ extra_keys_with_types(obj, name, suffix)|indent }} +{% endif %} +{% endfor %} +{%- endmacro -%} + +{%- macro extra_keys_with_types(obj, name, suffix) -%} +#[derive(Default, Debug, Clone, Hash, Eq, PartialEq)] +pub struct {{ obj.name|Camelize }}{{ suffix }} { + {% for item, type in obj|attr(name) %} + pub {{ item|snake_case }}: Option<{{type|extra_type_name}}>, + {% endfor %} +} + +impl ExtraKeys for {{ obj.name|Camelize }}{{ suffix }} { + const ALLOWED_KEYS: &'static [&'static str] = {{ obj.allowed_extra_keys|extra_keys }}; + + fn into_ffi_extra(self) -> ::std::collections::HashMap<String, String> { + let mut map = ::std::collections::HashMap::new(); + {% for key, _ in obj|attr(name) %} + self.{{key|snake_case}}.and_then(|val| map.insert("{{key|snake_case}}".into(), val.to_string())); + {% endfor %} + map + } +} +{%- endmacro -%} + +{% macro generate_label_enum(obj) %} +#[repr(u16)] +pub enum {{ obj.name|Camelize }}Label { + {% for label in obj.ordered_labels %} + {# Specifically _not_ using r# as C++ doesn't have it #} + {{ label|Camelize }} = {{loop.index0}}, + {% endfor %} + __Other__, +} +impl From<u16> for {{ obj.name|Camelize }}Label { + fn from(v: u16) -> Self { + match v { + {% for label in obj.ordered_labels %} + {{ loop.index0 }} => Self::{{ label|Camelize }}, + {% endfor %} + _ => Self::__Other__, + } + } +} +impl Into<&'static str> for {{ obj.name|Camelize }}Label { + fn into(self) -> &'static str { + match self { + {% for label in obj.ordered_labels %} + Self::{{ label| Camelize }} => "{{label}}", + {% endfor %} + Self::__Other__ => "__other__", + } + } +} +{%- endmacro -%} + +pub enum DynamicLabel { } + +{% for category_name, objs in all_objs.items() %} +pub mod {{ category_name|snake_case }} { + use crate::private::*; + #[allow(unused_imports)] // CommonMetricData might be unused, let's avoid warnings + use glean::CommonMetricData; + #[allow(unused_imports)] // HistogramType might be unusued, let's avoid warnings + use glean::HistogramType; + use once_cell::sync::Lazy; + + {% for obj in objs.values() %} + {% if obj|attr("_generate_enums") %} +{{ generate_extra_keys(obj) }} + {%- endif %} + {% if obj.labeled and obj.labels and obj.labels|length %} + {{ generate_label_enum(obj)|indent }} + {% endif %} + #[allow(non_upper_case_globals)] + /// generated from {{ category_name }}.{{ obj.name }} + /// + /// {{ obj.description|wordwrap() | replace('\n', '\n /// ') }} + {% if obj.type == "counter" and obj.send_in_pings|length == 1 and not obj.disabled and obj.lifetime|rust == "Lifetime::Ping" %} + {# Use optimized CounterMetric ctor in a common case (esp. for Use Counters) #} + pub static {{ obj.name|snake_case }}: Lazy<{{ obj|type_name }}> = Lazy::new(|| { + CounterMetric::codegen_new( + {{obj|metric_id}}, + "{{obj.category}}", + "{{obj.name}}", + "{{obj.send_in_pings[0]}}" + ) + }); + {% else %} + pub static {{ obj.name|snake_case }}: Lazy<{{ obj|type_name }}> = Lazy::new(|| { + {{ obj|ctor }}({{obj|metric_id}}.into(), CommonMetricData { + {% for arg_name in common_metric_data_args if obj[arg_name] is defined %} + {{ arg_name }}: {{ obj[arg_name]|rust }}, + {% endfor %} + ..Default::default() + } + {%- for arg_name in extra_args if obj[arg_name] is defined and arg_name not in common_metric_data_args and arg_name != 'allowed_extra_keys' -%} + , {{ obj[arg_name]|rust }} + {%- endfor -%} + {{ ", " if obj.labeled else ")\n" }} + {%- if obj.labeled -%} + {%- if obj.labels -%} + Some({{ obj.labels|rust }}) + {%- else -%} + None + {%- endif -%}) + {% endif %} + }); + {% endif %} + + {% endfor %} +} +{% endfor %} + +{% if metric_by_type|length > 0 %} +#[allow(dead_code)] +pub(crate) mod __glean_metric_maps { + use std::collections::HashMap; + + use crate::metrics::extra_keys_len; + use crate::private::*; + use once_cell::sync::Lazy; + +{% for typ, metrics in metric_by_type.items() %} + pub static {{typ.0}}: Lazy<HashMap<MetricId, &Lazy<{{typ.1}}>>> = Lazy::new(|| { + let mut map = HashMap::with_capacity({{metrics|length}}); + {% for metric in metrics %} + map.insert({{metric.0}}.into(), &super::{{metric.1}}); + {% endfor %} + map + }); + +{% endfor %} + + /// Wrapper to record an event based on its metric ID. + /// + /// # Arguments + /// + /// * `metric_id` - The metric's ID to look up + /// * `extra` - An map of (extra key id, string) pairs. + /// The map will be decoded into the appropriate `ExtraKeys` type. + /// # Returns + /// + /// Returns `Ok(())` if the event was found and `record` was called with the given `extra`, + /// or an `EventRecordingError::InvalidId` if no event by that ID exists + /// or an `EventRecordingError::InvalidExtraKey` if the `extra` map could not be deserialized. + pub(crate) fn record_event_by_id(metric_id: u32, extra: HashMap<String, String>) -> Result<(), EventRecordingError> { + match metric_id { +{% for metric_id, event in events_by_id.items() %} + {{metric_id}} => { + assert!( + extra_keys_len(&super::{{event}}) != 0 || extra.is_empty(), + "No extra keys allowed, but some were passed" + ); + + super::{{event}}.record_raw(extra); + Ok(()) + } +{% endfor %} + _ => Err(EventRecordingError::InvalidId), + } + } + + /// Wrapper to record an event based on its metric ID, with a provided timestamp. + /// + /// # Arguments + /// + /// * `metric_id` - The metric's ID to look up + /// * `timestamp` - The time at which this event was recorded. + /// * `extra` - An map of (extra key id, string) pairs. + /// The map will be decoded into the appropriate `ExtraKeys` type. + /// # Returns + /// + /// Returns `Ok(())` if the event was found and `record` was called with the given `extra`, + /// or an `EventRecordingError::InvalidId` if no event by that ID exists + /// or an `EventRecordingError::InvalidExtraKey` if the event doesn't take extra pairs, + /// but some are passed in. + pub(crate) fn record_event_by_id_with_time(metric_id: MetricId, timestamp: u64, extra: HashMap<String, String>) -> Result<(), EventRecordingError> { + match metric_id { +{% for metric_id, event in events_by_id.items() %} + MetricId({{metric_id}}) => { + if extra_keys_len(&super::{{event}}) == 0 && !extra.is_empty() { + return Err(EventRecordingError::InvalidExtraKey); + } + + super::{{event}}.record_with_time(timestamp, extra); + Ok(()) + } +{% endfor %} + _ => Err(EventRecordingError::InvalidId), + } + } + + /// Wrapper to get the currently stored events for event metric. + /// + /// # Arguments + /// + /// * `metric_id` - The metric's ID to look up + /// * `ping_name` - (Optional) The ping name to look into. + /// Defaults to the first value in `send_in_pings`. + /// + /// # Returns + /// + /// Returns the recorded events or `None` if nothing stored. + /// + /// # Panics + /// + /// Panics if no event by the given metric ID could be found. + pub(crate) fn event_test_get_value_wrapper(metric_id: u32, ping_name: Option<String>) -> Option<Vec<RecordedEvent>> { + match metric_id { +{% for metric_id, event in events_by_id.items() %} + {{metric_id}} => super::{{event}}.test_get_value(ping_name.as_deref()), +{% endfor %} + _ => panic!("No event for metric id {}", metric_id), + } + } + + /// Check the provided event for errors. + /// + /// # Arguments + /// + /// * `metric_id` - The metric's ID to look up + /// * `ping_name` - (Optional) The ping name to look into. + /// Defaults to the first value in `send_in_pings`. + /// + /// # Returns + /// + /// Returns a string for the recorded error or `None`. + /// + /// # Panics + /// + /// Panics if no event by the given metric ID could be found. + #[allow(unused_variables)] + pub(crate) fn event_test_get_error(metric_id: u32) -> Option<String> { + #[cfg(feature = "with_gecko")] + match metric_id { +{% for metric_id, event in events_by_id.items() %} + {{metric_id}} => test_get_errors!(super::{{event}}), +{% endfor %} + _ => panic!("No event for metric id {}", metric_id), + } + + #[cfg(not(feature = "with_gecko"))] + { + return None; + } + } + +{% for labeled_type, labeleds_by_id in labeleds_by_id_by_type.items() %} + /// Gets the submetric from the specified labeled_{{labeled_type}} metric. + /// + /// # Arguments + /// + /// * `metric_id` - The metric's ID to look up + /// * `label` - The label identifying the {{labeled_type}} submetric. + /// + /// # Returns + /// + /// Returns the {{labeled_type}} submetric. + /// + /// # Panics + /// + /// Panics if no labeled_{{labeled_type}} by the given metric ID could be found. + #[allow(unused_variables)] + pub(crate) fn labeled_{{labeled_type}}_get(metric_id: u32, label: &str) -> Labeled{{labeled_type|Camelize}}Metric { + match metric_id { +{% for metric_id, (labeled, _) in labeleds_by_id.items() %} + {{metric_id}} => super::{{labeled}}.get(label), +{% endfor %} + _ => panic!("No labeled_{{labeled_type}} for metric id {}", metric_id), + } + } + + /// Gets the submetric from the specified labeled_{{labeled_type}} metric, by enum. + /// + /// # Arguments + /// + /// * `metric_id` - The metric's ID to look up + /// * `label_enum` - The label enum identifying the {{labeled_type}} submetric. + /// + /// # Returns + /// + /// Returns the {{labeled_type}} submetric. + /// + /// # Panics + /// + /// Panics if no labeled_{{labeled_type}} by the given metric ID could be found. + #[allow(unused_variables)] + pub(crate) fn labeled_{{labeled_type}}_enum_get(metric_id: u32, label_enum: u16) -> Labeled{{labeled_type|Camelize}}Metric { + match metric_id { +{% for metric_id, (labeled, has_enum) in labeleds_by_id.items() %} +{% if has_enum %} + {{metric_id}} => super::{{labeled}}.get(labeled_enum_to_str(metric_id, label_enum)), +{% endif %} +{% endfor %} + _ => panic!("No labeled_{{labeled_type}} for metric id {}", metric_id), + } + } +{% endfor %} + + pub(crate) fn labeled_enum_to_str(metric_id: u32, label: u16) -> &'static str { + match metric_id { +{% for category_name, objs in all_objs.items() %} +{% for obj in objs.values() %} +{% if obj.labeled and obj.labels and obj.labels|length %} + {{obj|metric_id}} => super::{{category_name|snake_case}}::{{obj.name|Camelize}}Label::from(label).into(), +{% endif %} +{% endfor %} +{% endfor %} + _ => panic!("Can't turn label enum to string for metric {} which isn't a labeled metric with static labels", metric_id), + } + } + + pub(crate) mod submetric_maps { + use std::sync::{ + atomic::AtomicU32, + RwLock, + }; + use super::*; + + pub(crate) const SUBMETRIC_BIT: u32 = {{submetric_bit}}; + pub(crate) static NEXT_LABELED_SUBMETRIC_ID: AtomicU32 = AtomicU32::new((1 << SUBMETRIC_BIT) + 1); + pub(crate) static LABELED_METRICS_TO_IDS: Lazy<RwLock<HashMap<(u32, String), u32>>> = Lazy::new(|| + RwLock::new(HashMap::new()) + ); + pub(crate) static LABELED_ENUMS_TO_IDS: Lazy<RwLock<HashMap<(u32, u16), u32>>> = Lazy::new(|| + RwLock::new(HashMap::new()) + ); + +{% for typ, metrics in metric_by_type.items() %} +{% if typ.0 in ('BOOLEAN_MAP', 'COUNTER_MAP', 'STRING_MAP') %} + pub static {{typ.0}}: Lazy<RwLock<HashMap<MetricId, Labeled{{typ.1}}>>> = Lazy::new(|| + RwLock::new(HashMap::new()) + ); +{% endif %} +{% endfor%} + } +} +{% endif %} diff --git a/toolkit/components/glean/build_scripts/glean_parser_ext/templates/rust_pings.jinja2 b/toolkit/components/glean/build_scripts/glean_parser_ext/templates/rust_pings.jinja2 new file mode 100644 index 0000000000..c041f663a6 --- /dev/null +++ b/toolkit/components/glean/build_scripts/glean_parser_ext/templates/rust_pings.jinja2 @@ -0,0 +1,77 @@ +// -*- mode: Rust -*- + +// AUTOGENERATED BY glean_parser. DO NOT EDIT. +{# The rendered source is autogenerated, but this +Jinja2 template is not. Please file bugs! #} + +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use crate::private::Ping; +use once_cell::sync::Lazy; + +{% for obj in all_objs['pings'].values() %} +#[allow(non_upper_case_globals)] +/// {{ obj.description|wordwrap() | replace('\n', '\n/// ') }} +pub static {{ obj.name|snake_case }}: Lazy<Ping> = Lazy::new(|| { + Ping::new( + "{{ obj.name }}", + {{ obj.include_client_id|rust }}, + {{ obj.send_if_empty|rust }}, + {{ obj.precise_timestamps|rust }}, + {{ obj.reason_codes|rust }}, + ) +}); + +{% endfor %} + +/// Instantiate custom pings once to trigger registration. +/// +/// # Arguments +/// +/// application_id: If present, limit to only registering custom pings +/// assigned to the identified application. +#[doc(hidden)] +pub fn register_pings(application_id: Option<&str>) { + match application_id { + {% for id, ping_names in ping_names_by_app_id.items() %} + Some("{{id}}") => { + log::info!("Registering pings {{ ping_names|join(', ') }} for {{id}}"); + {% for ping_name in ping_names %} + let _ = &*{{ ping_name|snake_case }}; + {% endfor %} + }, + {% endfor %} + _ => { + {% for obj in all_objs['pings'].values() %} + let _ = &*{{ obj.name|snake_case }}; + {% endfor %} + } + } +} + +#[cfg(feature = "with_gecko")] +pub(crate) fn submit_ping_by_id(id: u32, reason: Option<&str>) { + if id & (1 << crate::factory::DYNAMIC_PING_BIT) > 0 { + let map = crate::factory::__jog_metric_maps::PING_MAP + .read() + .expect("Read lock for dynamic ping map was poisoned!"); + if let Some(ping) = map.get(&id) { + ping.submit(reason); + } else { + // TODO: instrument this error. + log::error!("Cannot submit unknown dynamic ping {} by id.", id); + } + return; + } + match id { +{% for obj in all_objs['pings'].values() %} + {{ obj.name|ping_id }} => {{ obj.name | snake_case }}.submit(reason), +{% endfor %} + _ => { + // TODO: instrument this error. + log::error!("Cannot submit unknown ping {} by id.", id); + } + } +} diff --git a/toolkit/components/glean/build_scripts/glean_parser_ext/util.py b/toolkit/components/glean/build_scripts/glean_parser_ext/util.py new file mode 100644 index 0000000000..722a3e409c --- /dev/null +++ b/toolkit/components/glean/build_scripts/glean_parser_ext/util.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Utility functions for the glean_parser-based code generator +""" +import copy +from typing import Dict, List, Tuple + +from glean_parser import util + + +def generate_ping_ids(objs): + """ + Return a lookup function for ping IDs per ping name. + + :param objs: A tree of objects as returned from `parser.parse_objects`. + """ + + if "pings" not in objs: + + def no_ping_ids_for_you(): + assert False + + return no_ping_ids_for_you + + # Ping ID 0 is reserved (but unused) right now. + ping_id = 1 + + ping_id_mapping = {} + for ping_name in objs["pings"].keys(): + ping_id_mapping[ping_name] = ping_id + ping_id += 1 + + return lambda ping_name: ping_id_mapping[ping_name] + + +def generate_metric_ids(objs): + """ + Return a lookup function for metric IDs per metric object. + + :param objs: A tree of metrics as returned from `parser.parse_objects`. + """ + + # Metric ID 0 is reserved (but unused) right now. + metric_id = 1 + + # Mapping from a tuple of (category name, metric name) to the metric's numeric ID + metric_id_mapping = {} + for category_name, metrics in objs.items(): + for metric in metrics.values(): + metric_id_mapping[(category_name, metric.name)] = metric_id + metric_id += 1 + + return lambda metric: metric_id_mapping[(metric.category, metric.name)] + + +def get_metrics(objs): + """ + Returns *just* the metrics in a set of Glean objects + """ + ret = copy.copy(objs) + for category in ["pings", "tags"]: + if ret.get(category): + del ret[category] + return ret + + +def type_ids_and_categories(objs) -> Tuple[Dict[str, Tuple[int, List[str]]], List[str]]: + """ + Iterates over the metrics in objs, constructing two metadata structures: + - metric_types: Dict[str, Tuple[int, List[str]]] - map from a metric + type (snake_case) to its metric type id and ordered list of arguments. + - categories: List[str] - category names (snake_case) + + Is stable across invocations: Will generate same ids for same objs. + (If it doesn't, JOG's factory disagreeing with GleanJSMetricsLookup + will break the build). + Uses the same order of metric args set out in glean_parser.util's + common_metric_args and extra_metric_args. + (If it didn't, it would supply args in the wrong order to metric type + constructors with multiple extra args (e.g. custom_distribution)). + """ + metric_type_ids = {} + categories = [] + + for category_name, objs in get_metrics(objs).items(): + categories.append(category_name) + + for metric in objs.values(): + if metric.type not in metric_type_ids: + type_id = len(metric_type_ids) + 1 + args = util.common_metric_args.copy() + for arg_name in util.extra_metric_args: + if hasattr(metric, arg_name): + args.append(arg_name) + metric_type_ids[metric.type] = {"id": type_id, "args": args} + + return (metric_type_ids, categories) diff --git a/toolkit/components/glean/build_scripts/mach_commands.py b/toolkit/components/glean/build_scripts/mach_commands.py new file mode 100644 index 0000000000..d385e53605 --- /dev/null +++ b/toolkit/components/glean/build_scripts/mach_commands.py @@ -0,0 +1,227 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from mach.decorators import Command, CommandArgument + +LICENSE_HEADER = """# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" + +GENERATED_HEADER = """ +### This file was AUTOMATICALLY GENERATED by `./mach update-glean-tags` +### DO NOT edit it by hand. +""" + + +@Command( + "data-review", + category="misc", + description="Generate a skeleton data review request form for a given bug's data", +) +@CommandArgument( + "bug", default=None, nargs="?", type=str, help="bug number or search pattern" +) +def data_review(command_context, bug=None): + # Get the metrics_index's list of metrics indices + # by loading the index as a module. + import sys + from os import path + + sys.path.append(path.join(path.dirname(__file__), path.pardir)) + from pathlib import Path + + from glean_parser import data_review + from metrics_index import metrics_yamls + + return data_review.generate( + bug, [Path(command_context.topsrcdir) / x for x in metrics_yamls] + ) + + +@Command( + "perf-data-review", + category="misc", + description="Generate a skeleton performance data review request form for a given bug's data", +) +@CommandArgument( + "bug", default=None, nargs="?", type=str, help="bug number or search pattern" +) +def perf_data_review(command_context, bug=None): + # Get the metrics_index's list of metrics indices + # by loading the index as a module. + import sys + from os import path + + sys.path.append(path.join(path.dirname(__file__), path.pardir)) + from metrics_index import metrics_yamls + + sys.path.append(path.dirname(__file__)) + from pathlib import Path + + import perf_data_review + + return perf_data_review.generate( + bug, [Path(command_context.topsrcdir) / x for x in metrics_yamls] + ) + + +@Command( + "update-glean-tags", + category="misc", + description=( + "Creates a list of valid glean tags based on in-tree bugzilla component definitions" + ), +) +def update_glean_tags(command_context): + from pathlib import Path + + import yaml + from mozbuild.backend.configenvironment import ConfigEnvironment + from mozbuild.frontend.reader import BuildReader + + config = ConfigEnvironment( + command_context.topsrcdir, + command_context.topobjdir, + defines=command_context.defines, + substs=command_context.substs, + ) + + reader = BuildReader(config) + bug_components = set() + for p in reader.read_topsrcdir(): + if p.get("BUG_COMPONENT"): + bug_components.add(p["BUG_COMPONENT"]) + + tags_filename = (Path(__file__).parent / "../tags.yaml").resolve() + + tags = {"$schema": "moz://mozilla.org/schemas/glean/tags/1-0-0"} + for bug_component in bug_components: + product = bug_component.product.strip() + component = bug_component.component.strip() + tags["{} :: {}".format(product, component)] = { + "description": "The Bugzilla component which applies to this object." + } + + open(tags_filename, "w").write( + "{}\n{}\n\n".format(LICENSE_HEADER, GENERATED_HEADER) + + yaml.dump(tags, width=78, explicit_start=True) + ) + + +def replace_in_file(path, pattern, replace): + """ + Replace `pattern` with `replace` in the file `path`. + The file is modified on disk. + + Returns `True` if exactly one replacement happened. + `False` otherwise. + """ + + import re + + with open(path, "r+") as file: + data = file.read() + data, subs_made = re.subn(pattern, replace, data, flags=re.MULTILINE) + + file.seek(0) + file.write(data) + file.truncate() + + if subs_made != 1: + return False + + return True + + +def replace_in_file_or_die(path, pattern, replace): + """ + Replace `pattern` with `replace` in the file `path`. + The file is modified on disk. + + If not exactly one occurrence of `pattern` was replaced it will exit with exit code 1. + """ + + import sys + + success = replace_in_file(path, pattern, replace) + if not success: + print(f"ERROR: Failed to replace one occurrence in {path}") + print(f" Pattern: {pattern}") + print(f" Replace: {replace}") + print("File was modified. Check the diff.") + sys.exit(1) + + +@Command( + "update-glean", + category="misc", + description="Update Glean to the given version", +) +@CommandArgument("version", help="Glean version to upgrade to") +def update_glean(command_context, version): + import textwrap + from pathlib import Path + + topsrcdir = Path(command_context.topsrcdir) + + replace_in_file_or_die( + topsrcdir / "build.gradle", + r'gleanVersion = "[0-9.]+"', + f'gleanVersion = "{version}"', + ) + replace_in_file_or_die( + topsrcdir / "toolkit" / "components" / "glean" / "Cargo.toml", + r'^glean = "[0-9.]+"', + f'glean = "{version}"', + ) + replace_in_file_or_die( + topsrcdir / "toolkit" / "components" / "glean" / "api" / "Cargo.toml", + r'^glean = "[0-9.]+"', + f'glean = "{version}"', + ) + replace_in_file_or_die( + topsrcdir / "gfx" / "wr" / "webrender" / "Cargo.toml", + r'^glean = "[0-9.]+"', + f'glean = "{version}"', + ) + replace_in_file_or_die( + topsrcdir / "python" / "sites" / "mach.txt", + r"glean-sdk==[0-9.]+", + f"glean-sdk=={version}", + ) + + instructions = f""" + We've edited most of the necessary files to require Glean SDK {version}. + + You will have to edit the following files yourself: + + gfx/wr/wr_glyph_rasterizer/Cargo.toml + + Then, to ensure Glean and Firefox's other Rust dependencies are appropriately vendored, + please run the following commands: + + cargo update -p glean + ./mach vendor rust --ignore-modified + + `./mach vendor rust` may identify version mismatches. + Please consult the Updating the Glean SDK docs for assistance: + https://firefox-source-docs.mozilla.org/toolkit/components/glean/dev/updating_sdk.html + + The Glean SDK is already vetted and no additional vetting for it is necessary. + To prune the configuration file after vendoring run: + + ./mach cargo vet prune + + Then, to update webrender which independently relies on the Glean SDK, run: + + cd gfx/wr + cargo update -p glean + + Then, to ensure all is well, build Firefox and run the FOG tests. + Instructions can be found here: + https://firefox-source-docs.mozilla.org/toolkit/components/glean/dev/testing.html + """ + + print(textwrap.dedent(instructions)) diff --git a/toolkit/components/glean/build_scripts/perf_data_review.py b/toolkit/components/glean/build_scripts/perf_data_review.py new file mode 100644 index 0000000000..8c84249a2a --- /dev/null +++ b/toolkit/components/glean/build_scripts/perf_data_review.py @@ -0,0 +1,168 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Produce skeleton Performance Data Review Requests. + +This was mostly copies from glean_parser, and should be kept in sync. +""" + +import re +from pathlib import Path +from typing import Sequence + +from glean_parser import parser, util + + +def generate( + bug: str, + metrics_files: Sequence[Path], +) -> int: + """ + Commandline helper for Data Review Request template generation. + + :param bug: pattern to match in metrics' bug_numbers lists. + :param metrics_files: List of Path objects to load metrics from. + :return: Non-zero if there were any errors. + """ + + metrics_files = util.ensure_list(metrics_files) + + # Accept any value of expires. + parser_options = { + "allow_reserved": True, + "custom_is_expired": lambda expires: False, + "custom_validate_expires": lambda expires: True, + } + all_objects = parser.parse_objects(metrics_files, parser_options) + + if util.report_validation_errors(all_objects): + return 1 + + # I tried [\W\Z] but it complained. So `|` it is. + reobj = re.compile(f"\\W{bug}\\W|\\W{bug}$") + durations = set() + responsible_emails = set() + metrics_table = "" + for category_name, metrics in all_objects.value.items(): + for metric in metrics.values(): + if not any([len(reobj.findall(bug)) == 1 for bug in metric.bugs]): + continue + + metric_name = util.snake_case(metric.name) + category_name = util.snake_case(category_name) + one_line_desc = metric.description.replace("\n", " ") + sensitivity = ", ".join([s.name for s in metric.data_sensitivity]) + last_bug = metric.bugs[-1] + metrics_table += f"`{category_name}.{metric_name}` | " + metrics_table += f"{one_line_desc} | {sensitivity} | {last_bug}\n" + if metric.type == "event" and len(metric.allowed_extra_keys): + for extra_name, extra_detail in metric.extra_keys.items(): + extra_one_line_desc = extra_detail["description"].replace("\n", " ") + metrics_table += f"`{category_name}.{metric_name}#{extra_name}` | " + metrics_table += ( + f"{extra_one_line_desc} | {sensitivity} | {last_bug}\n" + ) + + durations.add(metric.expires) + + if metric.expires == "never": + responsible_emails.update(metric.notification_emails) + + if len(durations) == 1: + duration = next(iter(durations)) + if duration == "never": + collection_duration = "This collection will be collected permanently." + else: + collection_duration = f"This collection has expiry '{duration}'" + else: + collection_duration = "Parts of this collection expire at different times: " + collection_duration += f"{durations}" + + if "never" in durations: + collection_duration += "\n" + ", ".join(responsible_emails) + " " + collection_duration += "will be responsible for the permanent collections." + + if len(durations) == 0: + print(f"I'm sorry, I couldn't find metrics matching the bug number {bug}.") + return 1 + + # This template is pulled from + # https://github.com/mozilla/data-review/blob/main/request.md + print( + """ +!! Reminder: it is your responsibility to complete and check the correctness of +!! this automatically-generated request skeleton before requesting Data +!! Collection Review. See https://wiki.mozilla.org/Data_Collection for details. + +DATA REVIEW REQUEST +1. What questions will you answer with this data? + +TODO: Fill this in. + +2. Why does Mozilla need to answer these questions? Are there benefits for users? + Do we need this information to address product or business requirements? + +In order to guarantee the performance of our products, it is vital to monitor +real-world installs used by real-world users. + +3. What alternative methods did you consider to answer these questions? + Why were they not sufficient? + +Our ability to measure the practical performance impact of changes through CI +and manual testing is limited. Monitoring the performance of our products in +the wild among real users is the only way to be sure we have an accurate +picture. + +4. Can current instrumentation answer these questions? + +No. + +5. List all proposed measurements and indicate the category of data collection for each + measurement, using the Firefox data collection categories found on the Mozilla wiki. + +Measurement Name | Measurement Description | Data Collection Category | Tracking Bug +---------------- | ----------------------- | ------------------------ | ------------""" + ) + print(metrics_table) + print( + """ +6. Please provide a link to the documentation for this data collection which + describes the ultimate data set in a public, complete, and accurate way. + +This collection is Glean so is documented +[in the Glean Dictionary](https://dictionary.telemetry.mozilla.org). + +7. How long will this data be collected? +""" + ) + print(collection_duration) + print( + """ +8. What populations will you measure? + +All channels, countries, and locales. No filters. + +9. If this data collection is default on, what is the opt-out mechanism for users? + +These collections are Glean. The opt-out can be found in the product's preferences. + +10. Please provide a general description of how you will analyze this data. + +This will be continuously monitored for regression and improvement detection. + +11. Where do you intend to share the results of your analysis? + +Internal monitoring (GLAM, Redash, Looker, etc.). + +12. Is there a third-party tool (i.e. not Telemetry) that you + are proposing to use for this data collection? + +No. +""" + ) + + return 0 |