7 files changed, 504 insertions, 0 deletions
diff --git a/mobile/android/fenix/tools/.gitignore b/mobile/android/fenix/tools/.gitignore
new file mode 100644
index 0000000000..2a20e4947e
--- /dev/null
+++ b/mobile/android/fenix/tools/.gitignore
@@ -0,0 +1,2 @@
+*_expiry_list.csv
+*_renewal_request.txt
diff --git a/mobile/android/fenix/tools/data_renewal_generate.py b/mobile/android/fenix/tools/data_renewal_generate.py
new file mode 100755
index 0000000000..4a4f2390ae
--- /dev/null
+++ b/mobile/android/fenix/tools/data_renewal_generate.py
@@ -0,0 +1,187 @@
+#!/usr/bin/env python3
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+"""
+A script to help generate telemetry renewal csv and request template.
+This script also modifies metrics.yaml to mark soon to expired telemetry entries.
+"""
+
+import csv
+import json
+import os
+import sys
+
+import yaml
+from yaml.loader import FullLoader
+
+METRICS_FILENAME = "../app/metrics.yaml"
+NEW_METRICS_FILENAME = "../app/metrics_new.yaml"
+GLEAN_DICTIONARY_PREFIX = "https://dictionary.telemetry.mozilla.org/apps/fenix/metrics/"
+
+# This is to make sure we only write headers for the csv file once
+write_header = True
+# The number of soon to expired telemetry detected
+total_count = 0
+
+USAGE = """usage: ./{script_name} future_fenix_version_number"""
+
+# list of values that we care about
+_KEY_FILTER = [
+    "type",
+    "description",
+    "bugs",
+    "data_reviews",
+    "expires",
+]
+
+
+def response(last_key, content, expire_version, writer, renewal):
+    global write_header
+    global total_count
+    for key, value in content.items():
+        if (key == "$schema") or (key == "no_lint"):
+            continue
+        if key == "disabled":
+            continue
+
+        if ("expires" in value) and (
+            (value["expires"] == "never") or (not value["expires"] <= expire_version)
+        ):
+            continue
+
+        if key == "type":
+            remove_keys = []
+            for key in content.keys():
+                if key not in _KEY_FILTER:
+                    remove_keys.append(key)
+
+            for key in remove_keys:
+                content.pop(key)
+
+            content["bugs"] = content["bugs"][0]
+            content["data_reviews"] = content["data_reviews"][0]
+            total_count += 1
+
+            # name of the telemtry
+            dictionary_url = GLEAN_DICTIONARY_PREFIX + last_key.lstrip(".").replace(
+                ".", "_"
+            )
+            result = {
+                "#": total_count,
+                "name": last_key.lstrip("."),
+                "glean dictionary": dictionary_url,
+            }
+            result.update(content)
+
+            # add columns for product to fille out, these should always be added at the end
+            result.update({"keep(Y/N)": ""})
+            result.update({"new expiry version": ""})
+            result.update({"reason to extend": ""})
+
+            # output data-renewal request template
+            if write_header:
+                header = result.keys()
+                writer.writerow(header)
+                write_header = False
+                renewal.write("# Request for Data Collection Renewal\n")
+                renewal.write("### Renew for 1 year\n")
+                renewal.write("Total: TBD\n")
+                renewal.write("———\n")
+
+            writer.writerow(result.values())
+
+            renewal.write("`" + last_key.lstrip(".") + "`:\n")
+            renewal.write(
+                "1) Provide a link to the initial Data Collection Review Request for this collection.\n"
+            )
+            renewal.write("    - " + content["data_reviews"] + "\n")
+            renewal.write("\n")
+            renewal.write("2) When will this collection now expire?\n")
+            renewal.write("    - TBD\n")
+            renewal.write("\n")
+            renewal.write("3) Why was the initial period of collection insufficient?\n")
+            renewal.write("    - TBD\n")
+            renewal.write("\n")
+            renewal.write("———\n")
+            return
+
+        if type(value) is dict:
+            response(last_key + "." + key, value, expire_version, writer, renewal)
+
+
+with open(METRICS_FILENAME, "r") as f:
+    try:
+        arg1 = sys.argv[1]
+    except Exception:
+        print("usage is to include argument of the form `100`")
+        quit()
+
+    # parse metrics.yaml to json
+    write_header = True
+    data = yaml.load(f, Loader=FullLoader)
+    json_data = json.dumps(data)
+    content = json.loads(str(json_data))
+    csv_filename = arg1 + "_expiry_list.csv"
+    renewal_filename = arg1 + "_renewal_request.txt"
+    current_version = int(arg1)
+
+    # remove files created by last run if exists
+    if os.path.exists(csv_filename):
+        print("remove old csv file")
+        os.remove(csv_filename)
+
+    # remove files created by last run if exists
+    if os.path.exists(renewal_filename):
+        print("remove old renewal request template file")
+        os.remove(renewal_filename)
+
+    # remove files created by last run if exists
+    if os.path.exists(NEW_METRICS_FILENAME):
+        print("remove old metrics yaml file")
+        os.remove(NEW_METRICS_FILENAME)
+
+    data_file = open(csv_filename, "w")
+    csv_writer = csv.writer(data_file)
+    renewal_file = open(renewal_filename, "w")
+
+    response("", content, current_version, csv_writer, renewal_file)
+    renewal_file.close()
+    print("Completed")
+    print("Total count: " + str(total_count))
+
+    # Go through the metrics.yaml file to mark expired telemetry
+    verify_count = 0
+    f.seek(0, 0)
+    data = f.readlines()
+    with open(NEW_METRICS_FILENAME, "w") as f2:
+        for line in data:
+            if line.lstrip(" ").startswith("expires: ") and not (
+                line.lstrip(" ").startswith("expires: never")
+            ):
+                start_pos = len("expires: ")
+                version = int(line.lstrip(" ")[start_pos:])
+                if version <= current_version:
+                    verify_count += 1
+                    f2.writelines(
+                        line.rstrip("\n")
+                        + " /* TODO <"
+                        + str(verify_count)
+                        + "> require renewal */\n"
+                    )
+                else:
+                    f2.writelines(line)
+            else:
+                f2.writelines(line)
+        f2.close()
+
+        print("\n==============================")
+        if total_count != verify_count:
+            print("!!! Count check failed !!!")
+        else:
+            print("Count check passed")
+        print("==============================")
+
+        os.remove(METRICS_FILENAME)
+        os.rename(NEW_METRICS_FILENAME, METRICS_FILENAME)
diff --git a/mobile/android/fenix/tools/data_renewal_request.py b/mobile/android/fenix/tools/data_renewal_request.py
new file mode 100755
index 0000000000..5f14292b0d
--- /dev/null
+++ b/mobile/android/fenix/tools/data_renewal_request.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+"""
+A script to help generate a data review request comment. Once the CSV has been filled by product,
+copy the filled version into the tools directory and run this script from there to generate a filled
+renewal request data review comment.
+"""
+
+import csv
+import sys
+
+try:
+    version = sys.argv[1]
+except Exception:
+    print("usage is to include arguments of the form <version>")
+    quit()
+
+expiry_filename = version + "_expiry_list.csv"
+filled_renewal_filename = version + "_filled_renewal_request.txt"
+
+csv_reader = csv.DictReader(open(expiry_filename, "r"))
+output_string = ""
+total_count = 0
+updated_version = int(version) + 13
+for row in csv_reader:
+    if row["keep(Y/N)"] == "n":
+        continue
+    total_count += 1
+    output_string += f'` {row["name"]}`\n'
+    output_string += "1) Provide a link to the initial Data Collection Review Request for this collection.\n"
+    output_string += f'    - {eval(row["data_reviews"])[0]}\n'
+    output_string += "\n"
+    output_string += "2) When will this collection now expire?\n"
+    if len(row["new expiry version"]) == 0:
+        output_string += f"    - {updated_version}\n"
+    else:
+        output_string += f'    - {row["new expiry version"]}\n'
+
+    output_string += "\n"
+    output_string += "3) Why was the initial period of collection insufficient?\n"
+    output_string += f'    - {row["reason to extend"]}\n'
+    output_string += "\n"
+    output_string += "———\n"
+
+header = "# Request for Data Collection Renewal\n"
+header += "### Renew for 1 year\n"
+header += f"Total: {total_count}\n"
+header += "———\n\n"
+
+with open(filled_renewal_filename, "w+") as out:
+    out.write(header + output_string)
+    out.close()
diff --git a/mobile/android/fenix/tools/run_benchmark.py b/mobile/android/fenix/tools/run_benchmark.py
new file mode 100644
index 0000000000..527ef2c932
--- /dev/null
+++ b/mobile/android/fenix/tools/run_benchmark.py
@@ -0,0 +1,77 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+import argparse
+import os
+import subprocess
+import webbrowser
+
+DESCRIPTION = """ This script is made to run benchmark tests on Fenix. It'll open
+the JSON output file in firefox (or another browser of your choice if you pass the string in)
+"""
+
+ff_browser = "firefox"
+target_directory = "{cwd}/app/build/".format(cwd=os.getcwd())
+output_path = "/storage/emulated/0/benchmark/"
+output_file = "org.mozilla.fenix-benchmarkData.json"
+file_url = "file:///"
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description=DESCRIPTION)
+    parser.add_argument(
+        "class_to_test",
+        help="Path to the class to test. Format it as 'org.mozilla.fenix.[path_to_benchmark_test",
+    )
+    parser.add_argument(
+        "--open_file_in_browser",
+        help="Open the JSON file in the browser once the tests are done.",
+    )
+    return parser.parse_args()
+
+
+def run_benchmark(class_to_test):
+    args = ["./gradlew", "-Pbenchmark", "app:connectedCheck"]
+    if class_to_test:
+        args.append(
+            "-Pandroid.testInstrumentationRunnerArguments.class={clazz}".format(
+                clazz=class_to_test
+            )
+        )
+    subprocess.run(args, check=True, text=True)
+
+
+def fetch_benchmark_results():
+    subprocess.run(
+        ["adb", "pull", "{path}{file}".format(path=output_path, file=output_file)],
+        cwd=target_directory,
+        check=True,
+        text=True,
+    )
+    print(
+        "The benchmark results can be seen here: {file_path}".format(
+            file_path=os.path.abspath("./{file}".format(file=file_url))
+        )
+    )
+
+
+def open_in_browser():
+    abs_path = os.path.abspath(
+        "{target_directory}{file}".format(
+            target_directory=target_directory, file=output_file
+        )
+    )
+    webbrowser.get(ff_browser).open_new(file_url + abs_path)
+
+
+def main():
+    args = parse_args()
+    run_benchmark(args.class_to_test)
+    fetch_benchmark_results()
+    if args.open_file_in_browser:
+        open_in_browser()
+
+
+if __name__ == "__main__":
+    main()
diff --git a/mobile/android/fenix/tools/setup-startup-profiling.py b/mobile/android/fenix/tools/setup-startup-profiling.py
new file mode 100755
index 0000000000..d11fcdb0f2
--- /dev/null
+++ b/mobile/android/fenix/tools/setup-startup-profiling.py
@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+"""
+A script to set up startup profiling with the Firefox Profiler. See
+https://profiler.firefox.com/docs/#/./guide-remote-profiling?id=startup-profiling
+for more information.
+"""
+
+import argparse
+import os
+import tempfile
+from subprocess import run
+
+PATH_PREFIX = "/data/local/tmp"
+
+PROD_FENIX = "fenix"
+PROD_GVE = "geckoview_example"
+PRODUCTS = [PROD_FENIX, PROD_GVE]
+
+GV_CONFIG = b"""env:
+  MOZ_PROFILER_STARTUP: 1
+  MOZ_PROFILER_STARTUP_INTERVAL: 5
+  MOZ_PROFILER_STARTUP_FEATURES: js,stackwalk,leaf,screenshots,ipcmessages,java,cpu
+  MOZ_PROFILER_STARTUP_FILTERS: GeckoMain,Compositor,Renderer,IPDL Background
+"""
+
+
+def parse_args():
+    p = argparse.ArgumentParser(
+        description=(
+            "Easily enable start up profiling using the Firefox Profiler. Finish capturing the profile in "
+            "about:debugging on desktop. See "
+            "https://profiler.firefox.com/docs/#/./guide-remote-profiling?id=startup-profiling for "
+            "details."
+        )
+    )
+    p.add_argument(
+        "command",
+        choices=["activate", "deactivate"],
+        help=(
+            "whether to activate or deactive start up "
+            "profiling for the given release channel"
+        ),
+    )
+    p.add_argument(
+        "release_channel",
+        choices=["nightly", "beta", "release", "debug"],
+        help=(
+            "the release channel to "
+            "change the startup profiling state of the command on"
+        ),
+    )
+
+    p.add_argument(
+        "-p",
+        "--product",
+        choices=PRODUCTS,
+        default=PROD_FENIX,
+        help="which product to work on",
+    )
+    return p.parse_args()
+
+
+def push(id, filename):
+    config = tempfile.NamedTemporaryFile(delete=False)
+    try:
+        # I think the file needs to be closed to save its contents for adb push to
+        # work correctly so we close it here and later delete it manually.
+        with config.file as f:
+            f.write(GV_CONFIG)
+
+        print("Pushing {} to device.".format(filename))
+        run(["adb", "push", config.name, os.path.join(PATH_PREFIX, filename)])
+        run(["adb", "shell", "am", "set-debug-app", "--persistent", id])
+        print(
+            "\nStartup profiling enabled on all future start ups, possibly even after reinstall."
+        )
+        print("Call script with `deactivate` to disable it.")
+        print(
+            "DISABLE 'Remote debugging via USB' IN THE APP SETTINGS BEFORE STARTING THE APP & RE-ENABLE TO CAPTURE THE PROFILE.",
+            "This avoids the additional overhead added when 'Remote debugging via USB' is enabled during start up.",
+            sep=os.linesep,
+        )
+    finally:
+        os.remove(config.name)
+
+
+def remove(filename):
+    print("Removing {} from device.".format(filename))
+    run(["adb", "shell", "rm", PATH_PREFIX + "/" + filename])
+    run(["adb", "shell", "am", "clear-debug-app"])
+
+
+def convert_channel_to_id(product, channel):
+    if product == PROD_FENIX:
+        mapping = {
+            "release": "org.mozilla.firefox",
+            "beta": "org.mozilla.firefox_beta",
+            "nightly": "org.mozilla.fenix",
+            "debug": "org.mozilla.fenix.debug",
+        }
+        return mapping[channel]
+    elif product == PROD_GVE:
+        return "org.mozilla.geckoview_example"
+
+
+def main():
+    args = parse_args()
+
+    id = convert_channel_to_id(args.product, args.release_channel)
+    filename = id + "-geckoview-config.yaml"
+
+    if args.command == "activate":
+        push(id, filename)
+    elif args.command == "deactivate":
+        remove(filename)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/mobile/android/fenix/tools/setup.cfg b/mobile/android/fenix/tools/setup.cfg
new file mode 100644
index 0000000000..6224f31452
--- /dev/null
+++ b/mobile/android/fenix/tools/setup.cfg
@@ -0,0 +1,2 @@
+[pycodestyle]
+max-line-length = 120
diff --git a/mobile/android/fenix/tools/update-glean-tags.py b/mobile/android/fenix/tools/update-glean-tags.py
new file mode 100755
index 0000000000..c2f649832c
--- /dev/null
+++ b/mobile/android/fenix/tools/update-glean-tags.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+
+"""
+Scrapes GitHub labels for Fenix and generates a set of glean tags for use in metrics
+
+See https://mozilla.github.io/glean/book/reference/yaml/tags.html
+"""
+import urllib
+from pathlib import Path
+
+import requests
+import yaml
+
+LICENSE_HEADER = """# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+
+GENERATED_HEADER = """
+### This file was AUTOMATICALLY GENERATED by `./tools/update-glean-tags.py`
+### DO NOT edit it by hand.
+
+# Disable line-length rule because the links in the descriptions can be long
+# yamllint disable rule:line-length
+"""
+
+TAGS_FILENAME = (Path(__file__).parent / "../app/tags.yaml").resolve()
+
+labels = []
+page = 1
+while True:
+    more_labels = requests.get(
+        f"https://api.github.com/repos/mozilla-mobile/fenix/labels?per_page=100&page={page}"
+    ).json()
+    if not more_labels:
+        break
+    labels += more_labels
+    page += 1
+
+tags = {"$schema": "moz://mozilla.org/schemas/glean/tags/1-0-0"}
+for label in labels:
+    if label["name"].startswith("Feature:"):
+        abbreviated_label = label["name"].replace("Feature:", "")
+        url = (
+            "https://github.com/mozilla-mobile/fenix/issues?q="
+            + urllib.parse.quote_plus(f"label:{label['name']}")
+        )
+        label_description = (
+            (label["description"].strip() + ". ") if len(label["description"]) else ""
+        )
+        tags[abbreviated_label] = {
+            "description": f"{label_description}Corresponds to the [{label['name']}]({url}) label on GitHub."
+        }
+
+open(TAGS_FILENAME, "w").write(
+    "{}\n{}\n\n".format(LICENSE_HEADER, GENERATED_HEADER)
+    + yaml.dump(tags, width=78, explicit_start=True)
+)