Merging upstream version 126.0.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

7 files changed, 429 insertions, 0 deletions

diff --git a/mobile/android/focus-android/tools/data_renewal_generate.py b/mobile/android/focus-android/tools/data_renewal_generate.py
new file mode 100755
index 0000000000..43dba38541
--- /dev/null
+++ b/mobile/android/focus-android/tools/data_renewal_generate.py
@@ -0,0 +1,189 @@
+#!/usr/bin/env python3
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+"""
+A script to help generate telemetry renewal csv and request template.
+This script also modifies metrics.yaml to mark soon to expired telemetry entries.
+"""
+
+import csv
+import json
+import os
+import sys
+
+import yaml
+from yaml.loader import FullLoader
+
+METRICS_FILENAME = "../app/metrics.yaml"
+NEW_METRICS_FILENAME = "../app/metrics_new.yaml"
+GLEAN_DICTIONARY_PREFIX = (
+    "https://dictionary.telemetry.mozilla.org/apps/focus_android/metrics/"
+)
+
+# This is to make sure we only write headers for the csv file once
+write_header = True
+# The number of soon to expired telemetry detected
+total_count = 0
+
+USAGE = """usage: ./{script_name} future_fenix_version_number"""
+
+# list of values that we care about
+_KEY_FILTER = [
+    "type",
+    "description",
+    "bugs",
+    "data_reviews",
+    "expires",
+]
+
+
+def response(last_key, content, expire_version, writer, renewal):
+    global write_header
+    global total_count
+    for key, value in content.items():
+        if (key == "$schema") or (key == "no_lint"):
+            continue
+        if key == "disabled":
+            continue
+
+        if ("expires" in value) and (
+            (value["expires"] == "never") or (not value["expires"] <= expire_version)
+        ):
+            continue
+
+        if key == "type":
+            remove_keys = []
+            for key in content.keys():
+                if key not in _KEY_FILTER:
+                    remove_keys.append(key)
+
+            for key in remove_keys:
+                content.pop(key)
+
+            content["bugs"] = content["bugs"][0]
+            content["data_reviews"] = content["data_reviews"][0]
+            total_count += 1
+
+            # name of the telemtry
+            dictionary_url = GLEAN_DICTIONARY_PREFIX + last_key.lstrip(".").replace(
+                ".", "_"
+            )
+            result = {
+                "#": total_count,
+                "name": last_key.lstrip("."),
+                "glean dictionary": dictionary_url,
+            }
+            result.update(content)
+
+            # add columns for product to fille out, these should always be added at the end
+            result.update({"keep(Y/N)": ""})
+            result.update({"new expiry version": ""})
+            result.update({"reason to extend": ""})
+
+            # output data-renewal request template
+            if write_header:
+                header = result.keys()
+                writer.writerow(header)
+                write_header = False
+                renewal.write("# Request for Data Collection Renewal\n")
+                renewal.write("### Renew for 1 year\n")
+                renewal.write("Total: TBD\n")
+                renewal.write("———\n")
+
+            writer.writerow(result.values())
+
+            renewal.write("`" + last_key.lstrip(".") + "`:\n")
+            renewal.write(
+                "1) Provide a link to the initial Data Collection Review Request for this collection.\n"
+            )
+            renewal.write("    - " + content["data_reviews"] + "\n")
+            renewal.write("\n")
+            renewal.write("2) When will this collection now expire?\n")
+            renewal.write("    - TBD\n")
+            renewal.write("\n")
+            renewal.write("3) Why was the initial period of collection insufficient?\n")
+            renewal.write("    - TBD\n")
+            renewal.write("\n")
+            renewal.write("———\n")
+            return
+
+        if type(value) is dict:
+            response(last_key + "." + key, value, expire_version, writer, renewal)
+
+
+with open(METRICS_FILENAME, "r") as f:
+    try:
+        arg1 = sys.argv[1]
+    except Exception:
+        print("usage is to include argument of the form `100`")
+        quit()
+
+    # parse metrics.yaml to json
+    write_header = True
+    data = yaml.load(f, Loader=FullLoader)
+    json_data = json.dumps(data)
+    content = json.loads(str(json_data))
+    csv_filename = arg1 + "_expiry_list.csv"
+    renewal_filename = arg1 + "_renewal_request.txt"
+    current_version = int(arg1)
+
+    # remove files created by last run if exists
+    if os.path.exists(csv_filename):
+        print("remove old csv file")
+        os.remove(csv_filename)
+
+    # remove files created by last run if exists
+    if os.path.exists(renewal_filename):
+        print("remove old renewal request template file")
+        os.remove(renewal_filename)
+
+    # remove files created by last run if exists
+    if os.path.exists(NEW_METRICS_FILENAME):
+        print("remove old metrics yaml file")
+        os.remove(NEW_METRICS_FILENAME)
+
+    data_file = open(csv_filename, "w")
+    csv_writer = csv.writer(data_file)
+    renewal_file = open(renewal_filename, "w")
+
+    response("", content, current_version, csv_writer, renewal_file)
+    renewal_file.close()
+    print("Completed")
+    print("Total count: " + str(total_count))
+
+    # Go through the metrics.yaml file to mark expired telemetry
+    verify_count = 0
+    f.seek(0, 0)
+    data = f.readlines()
+    with open(NEW_METRICS_FILENAME, "w") as f2:
+        for line in data:
+            if line.lstrip(" ").startswith("expires: ") and not (
+                line.lstrip(" ").startswith("expires: never")
+            ):
+                start_pos = len("expires: ")
+                version = int(line.lstrip(" ")[start_pos:])
+                if version <= current_version:
+                    verify_count += 1
+                    f2.writelines(
+                        line.rstrip("\n")
+                        + " /* TODO <"
+                        + str(verify_count)
+                        + "> require renewal */\n"
+                    )
+                else:
+                    f2.writelines(line)
+            else:
+                f2.writelines(line)
+        f2.close()
+
+        print("\n==============================")
+        if total_count != verify_count:
+            print("!!! Count check failed !!!")
+        else:
+            print("Count check passed")
+        print("==============================")
+
+        os.remove(METRICS_FILENAME)
+        os.rename(NEW_METRICS_FILENAME, METRICS_FILENAME)
diff --git a/mobile/android/focus-android/tools/data_renewal_request.py b/mobile/android/focus-android/tools/data_renewal_request.py
new file mode 100755
index 0000000000..5f14292b0d
--- /dev/null
+++ b/mobile/android/focus-android/tools/data_renewal_request.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+"""
+A script to help generate a data review request comment. Once the CSV has been filled by product,
+copy the filled version into the tools directory and run this script from there to generate a filled
+renewal request data review comment.
+"""
+
+import csv
+import sys
+
+try:
+    version = sys.argv[1]
+except Exception:
+    print("usage is to include arguments of the form <version>")
+    quit()
+
+expiry_filename = version + "_expiry_list.csv"
+filled_renewal_filename = version + "_filled_renewal_request.txt"
+
+csv_reader = csv.DictReader(open(expiry_filename, "r"))
+output_string = ""
+total_count = 0
+updated_version = int(version) + 13
+for row in csv_reader:
+    if row["keep(Y/N)"] == "n":
+        continue
+    total_count += 1
+    output_string += f'` {row["name"]}`\n'
+    output_string += "1) Provide a link to the initial Data Collection Review Request for this collection.\n"
+    output_string += f'    - {eval(row["data_reviews"])[0]}\n'
+    output_string += "\n"
+    output_string += "2) When will this collection now expire?\n"
+    if len(row["new expiry version"]) == 0:
+        output_string += f"    - {updated_version}\n"
+    else:
+        output_string += f'    - {row["new expiry version"]}\n'
+
+    output_string += "\n"
+    output_string += "3) Why was the initial period of collection insufficient?\n"
+    output_string += f'    - {row["reason to extend"]}\n'
+    output_string += "\n"
+    output_string += "———\n"
+
+header = "# Request for Data Collection Renewal\n"
+header += "### Renew for 1 year\n"
+header += f"Total: {total_count}\n"
+header += "———\n\n"
+
+with open(filled_renewal_filename, "w+") as out:
+    out.write(header + output_string)
+    out.close()
diff --git a/mobile/android/focus-android/tools/docker/Dockerfile b/mobile/android/focus-android/tools/docker/Dockerfile
new file mode 100644
index 0000000000..f0a1be21a7
--- /dev/null
+++ b/mobile/android/focus-android/tools/docker/Dockerfile
@@ -0,0 +1,78 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Inspired by:
+# https://hub.docker.com/r/runmymind/docker-android-sdk/~/dockerfile/
+
+FROM ubuntu:22.04
+
+MAINTAINER Sebastian Kaspari "skaspari@mozilla.com"
+
+# -- System -----------------------------------------------------------------------------
+
+ENV GRADLE_OPTS='-Xmx4096m -Dorg.gradle.daemon=false' \
+    LANG='en_US.UTF-8' \
+    TERM='dumb' \
+    JAVA17PATH="/usr/lib/jvm/java-17-openjdk-amd64/bin/:$PATH"
+
+RUN apt-get update -qq \
+    # We need to install tzdata before all of the other packages. Otherwise it will show an interactive dialog
+    # which we cannot navigate while building the Docker image.
+    && apt-get install -y tzdata \
+    && apt-get install -y openjdk-17-jdk \
+                          git \
+                          curl \
+                          python3 \
+                          python-pip3 \
+                          locales \
+                          unzip \
+			  mercurial \
+    && apt-get clean
+
+# Today's Fastlane depends on a newer Ruby version than Ubuntu 17.10 has, so since
+# fastlane is only used for screenshots (afaik) just skip it.
+#RUN gem install fastlane
+
+RUN locale-gen en_US.UTF-8
+
+# -- Android SDK ------------------------------------------------------------------------
+
+RUN cd /opt && curl --location --retry 5 --output android-sdk.zip https://dl.google.com/android/repository/sdk-tools-linux-3859397.zip \
+  && unzip -d /opt/android-sdk-linux android-sdk.zip \
+  && rm -f android-sdk.zip
+
+ENV ANDROID_SDK_HOME /opt/android-sdk-linux
+ENV ANDROID_HOME /opt/android-sdk-linux
+
+RUN yes | PATH=$JAVA17PATH "${ANDROID_SDK_HOME}/cmdline-tools/bin/sdkmanager" --licenses
+
+# -- Project setup ----------------------------------------------------------------------
+
+WORKDIR /opt
+
+# Checkout source code
+RUN git clone https://github.com/mozilla-mobile/focus-android.git
+
+# Build project and run gradle tasks once to pull all dependencies
+WORKDIR /opt/focus-android
+RUN ./gradlew assembleFocusDebug \
+  && ./gradlew testFocusDebugUnitTest \
+  && ./gradlew detekt \
+  && ./gradlew ktlint \
+  && ./gradlew clean
+
+# -- Post setup -------------------------------------------------------------------------
+
+# Install taskcluster python library (used by decision tasks)
+# 5.0.0 is still incompatible with taskclusterProxy, meaning no decision task is able
+# to schedule the rest of the Taskcluster tasks. Please upgrade to taskcluster>=5 once
+# https://bugzilla.mozilla.org/show_bug.cgi?id=1460015 is fixed
+RUN pip install 'taskcluster>=4,<5'
+
+# Install Google Cloud SDK for using Firebase Test Lab
+RUN cd /opt && curl --location --retry 5 --output gcloud.tar.gz https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-331.0.0-linux-x86_64.tar.gz \
+  && tar -xvf /opt/gcloud.tar.gz \
+  && rm -f gcloud.tar.gz \
+  && /opt/google-cloud-sdk/install.sh --quiet \
+  && /opt/google-cloud-sdk/bin/gcloud --quiet components update
diff --git a/mobile/android/focus-android/tools/docker/licenses/android-sdk-license b/mobile/android/focus-android/tools/docker/licenses/android-sdk-license
new file mode 100644
index 0000000000..ff1da21494
--- /dev/null
+++ b/mobile/android/focus-android/tools/docker/licenses/android-sdk-license
@@ -0,0 +1,2 @@
+
+8933bad161af4178b1185d1a37fbf41ea5269c55
diff --git a/mobile/android/focus-android/tools/docker/licenses/android-sdk-preview-license b/mobile/android/focus-android/tools/docker/licenses/android-sdk-preview-license
new file mode 100644
index 0000000000..74069f8f02
--- /dev/null
+++ b/mobile/android/focus-android/tools/docker/licenses/android-sdk-preview-license
@@ -0,0 +1,2 @@
+
+84831b9409646a918e30573bab4c9c91346d8abd
diff --git a/mobile/android/focus-android/tools/gradle/versionCode.gradle b/mobile/android/focus-android/tools/gradle/versionCode.gradle
new file mode 100644
index 0000000000..73a4de1189
--- /dev/null
+++ b/mobile/android/focus-android/tools/gradle/versionCode.gradle
@@ -0,0 +1,45 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import java.text.SimpleDateFormat
+
+// This gradle scripts generates a "unique" version code for our release versions.
+//
+// The result of the version code depends on the timezone. We assume that this script will only be used
+// for release versions and running on our build servers with a fixed timezone.
+//
+// The version code is composed like: yDDDHHmm
+//  * y   = Double digit year, with 16 substracted: 2017 -> 17 -> 1
+//  * DDD = Day of the year, pad with zeros if needed: September 6th -> 249
+//  * HH  = Hour in day (00-23)
+//  * mm  = Minute in hour 
+//
+// For September 6th, 2017, 9:41 am this will generate the versionCode: 12490941 (1-249-09-41).
+//
+// Note that we only use this generated version code for builds we want to distribute. For local
+// debug builds we use a fixed versionCode to not mess with the caching mechanism of the build
+// system.
+
+ext {
+    def base = "3"
+    def today = new Date()
+
+    // We use the current year (double digit) and substract 16. We first released Focus in
+    // 2017 so this value will start counting at 1 and increment by one every year.
+    def year = String.valueOf((new SimpleDateFormat("yy").format(today) as int) - 16)
+
+    // We use the day in the Year (e.g. 248) as opposed to month + day (0510) because it's one digit shorter.
+    // If needed we pad with zeros (e.g. 25 -> 025)
+    def day = String.format("%03d", (new SimpleDateFormat("D").format(today) as int))
+ 
+    // We append the hour in day (24h) and minute in hour (7:26 pm -> 1926). We do not append
+    // seconds. This assumes that we do not need to build multiple release(!) builds the same
+    // minute.
+    def time = new SimpleDateFormat("HHmm").format(today)
+
+    generatedVersionCode = (base + year + day + time) as int
+
+    println("Generated versionCode: $generatedVersionCode")
+    println()
+}
diff --git a/mobile/android/focus-android/tools/update-glean-tags.py b/mobile/android/focus-android/tools/update-glean-tags.py
new file mode 100755
index 0000000000..8e77a8241b
--- /dev/null
+++ b/mobile/android/focus-android/tools/update-glean-tags.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+
+"""
+Scrapes GitHub labels for Focus and generates a set of glean tags for use in metrics
+
+See https://mozilla.github.io/glean/book/reference/yaml/tags.html
+"""
+import urllib
+from pathlib import Path
+
+import requests
+import yaml
+
+LICENSE_HEADER = """# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+
+GENERATED_HEADER = """
+### This file was AUTOMATICALLY GENERATED by `./tools/update-glean-tags.py`
+### DO NOT edit it by hand.
+
+# Disable line-length rule because the links in the descriptions can be long
+# yamllint disable rule:line-length
+"""
+
+TAGS_FILENAME = (Path(__file__).parent / "../app/tags.yaml").resolve()
+
+labels = []
+page = 1
+while True:
+    more_labels = requests.get(
+        f"https://api.github.com/repos/mozilla-mobile/focus-android/labels?per_page=100&page={page}"
+    ).json()
+    if not more_labels:
+        break
+    labels += more_labels
+    page += 1
+
+tags = {"$schema": "moz://mozilla.org/schemas/glean/tags/1-0-0"}
+for label in labels:
+    if label["name"].startswith("Feature:"):
+        abbreviated_label = label["name"].replace("Feature:", "")
+        url = (
+            "https://github.com/mozilla-mobile/focus-android/issues?q="
+            + urllib.parse.quote_plus(f"label:{label['name']}")
+        )
+        label_description = (
+            (label["description"].strip() + ". ") if len(label["description"]) else ""
+        )
+        tags[abbreviated_label] = {
+            "description": f"{label_description}Corresponds to the [{label['name']}]({url}) label on GitHub."
+        }
+
+open(TAGS_FILENAME, "w").write(
+    "{}\n{}\n\n".format(LICENSE_HEADER, GENERATED_HEADER)
+    + yaml.dump(tags, width=78, explicit_start=True)
+)