Adding upstream version 86.0.1.upstream/86.0.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 153 insertions, 0 deletions

diff --git a/build/submit_telemetry_data.py b/build/submit_telemetry_data.py
new file mode 100644
index 0000000000..0748fbfead
--- /dev/null
+++ b/build/submit_telemetry_data.py
@@ -0,0 +1,153 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import print_function
+
+import datetime
+import json
+import logging
+import os
+import sys
+
+import requests
+import voluptuous
+import voluptuous.humanize
+
+from mozbuild.telemetry import (
+    schema as build_telemetry_schema,
+    verify_statedir,
+)
+
+BUILD_TELEMETRY_URL = "https://incoming.telemetry.mozilla.org/{endpoint}"
+SUBMIT_ENDPOINT = "submit/eng-workflow/build/1/{ping_uuid}"
+STATUS_ENDPOINT = "status"
+
+
+def delete_expired_files(directory, days=30):
+    """Discards files in a directory older than a specified number
+    of days
+    """
+    now = datetime.datetime.now()
+    for filename in os.listdir(directory):
+        filepath = os.path.join(directory, filename)
+
+        ctime = os.path.getctime(filepath)
+        then = datetime.datetime.fromtimestamp(ctime)
+
+        if (now - then) > datetime.timedelta(days=days):
+            os.remove(filepath)
+
+    return
+
+
+def check_edge_server_status(session):
+    """Returns True if the Telemetry Edge Server
+    is ready to accept data
+    """
+    status_url = BUILD_TELEMETRY_URL.format(endpoint=STATUS_ENDPOINT)
+    response = session.get(status_url)
+    if response.status_code != 200:
+        return False
+    return True
+
+
+def send_telemetry_ping(session, data, ping_uuid):
+    """Sends a single build telemetry ping to the
+    edge server, returning the response object
+    """
+    resource_url = SUBMIT_ENDPOINT.format(ping_uuid=str(ping_uuid))
+    url = BUILD_TELEMETRY_URL.format(endpoint=resource_url)
+    response = session.post(url, json=data)
+
+    return response
+
+
+def submit_telemetry_data(outgoing, submitted):
+    """Sends information about `./mach build` invocations to
+    the Telemetry pipeline
+    """
+    with requests.Session() as session:
+        # Confirm the server is OK
+        if not check_edge_server_status(session):
+            logging.error('Error posting to telemetry: server status is not "200 OK"')
+            return 1
+
+        for filename in os.listdir(outgoing):
+            path = os.path.join(outgoing, filename)
+
+            if os.path.isdir(path) or not path.endswith(".json"):
+                logging.info("skipping item {}".format(path))
+                continue
+
+            ping_uuid = os.path.splitext(filename)[0]  # strip ".json" to get ping UUID
+
+            try:
+                with open(path, "r") as f:
+                    data = json.load(f)
+
+                # Verify the data matches the schema
+                voluptuous.humanize.validate_with_humanized_errors(
+                    data, build_telemetry_schema
+                )
+
+                response = send_telemetry_ping(session, data, ping_uuid)
+                if response.status_code != 200:
+                    msg = "response code {code} sending {uuid} to telemetry: {body}".format(
+                        body=response.content,
+                        code=response.status_code,
+                        uuid=ping_uuid,
+                    )
+                    logging.error(msg)
+                    continue
+
+                # Move from "outgoing" to "submitted"
+                os.rename(
+                    os.path.join(outgoing, filename), os.path.join(submitted, filename)
+                )
+
+                logging.info("successfully posted {} to telemetry".format(ping_uuid))
+
+            except ValueError as ve:
+                # ValueError is thrown if JSON cannot be decoded
+                logging.exception("exception parsing JSON at %s: %s" % (path, str(ve)))
+                os.remove(path)
+
+            except voluptuous.Error as e:
+                # Invalid is thrown if some data does not fit
+                # the correct Schema
+                logging.exception("invalid data found at %s: %s" % (path, e.message))
+                os.remove(path)
+
+            except Exception as e:
+                logging.error("exception posting to telemetry " "server: %s" % str(e))
+                break
+
+    delete_expired_files(submitted)
+
+    return 0
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("usage: python submit_telemetry_data.py <statedir>")
+        sys.exit(1)
+
+    statedir = sys.argv[1]
+
+    try:
+        outgoing, submitted, telemetry_log = verify_statedir(statedir)
+
+        # Configure logging
+        logging.basicConfig(
+            filename=telemetry_log,
+            format="%(asctime)s %(message)s",
+            level=logging.DEBUG,
+        )
+
+        sys.exit(submit_telemetry_data(outgoing, submitted))
+
+    except Exception as e:
+        # Handle and print messages from `statedir` verification
+        print(e.message)
+        sys.exit(1)