diff options
Diffstat (limited to 'dom/quota/scripts/fetch_qm_failures.py')
-rwxr-xr-x | dom/quota/scripts/fetch_qm_failures.py | 142 |
1 files changed, 0 insertions, 142 deletions
diff --git a/dom/quota/scripts/fetch_qm_failures.py b/dom/quota/scripts/fetch_qm_failures.py deleted file mode 100755 index 546b213582..0000000000 --- a/dom/quota/scripts/fetch_qm_failures.py +++ /dev/null @@ -1,142 +0,0 @@ -#!/usr/bin/env python3 -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import getopt -import sys - -import telemetry -import utils - -""" -The analysis is based on the following query: -https://sql.telemetry.mozilla.org/queries/78691/source?p_day=28&p_month=03&p_year=2021 - -SELECT UNIX_MILLIS(timestamp) AS submit_timeabs, - session_start_time, - submission_date, - build_id, - client_id, - session_id, - event_timestamp, - CAST(mozfun.map.get_key(event_map_values, "seq") AS INT64) AS seq, - mozfun.map.get_key(event_map_values, "context") AS context, - mozfun.map.get_key(event_map_values, "source_file") AS source_file, - mozfun.map.get_key(event_map_values, "source_line") AS source_line, - mozfun.map.get_key(event_map_values, "severity") AS severity, - mozfun.map.get_key(event_map_values, "result") AS result, -FROM telemetry.events -WHERE submission_date >= CAST('{{ year }}-{{ month }}-{{ day }}' AS DATE) - AND event_category='dom.quota.try' - AND build_id >= '{{ build }}' - AND UNIX_MILLIS(timestamp) > {{ last }} -ORDER BY submit_timeabs -LIMIT 600000 - -We fetch events in chronological order, as we want to keep track of where we already -arrived with our analysis. To accomplish this we write our runs into qmexecutions.json. - -[ - { - "workdir": ".", - "daysback": 1, - "numrows": 17377, - "lasteventtime": 1617303855145, - "rawfile": "./qmrows_until_1617303855145.json" - } -] - -lasteventtime is the highest value of event_timeabs we found in our data. - -analyze_qm_failures instead needs the rows to be ordered by -client_id, session_id, thread_id, submit_timeabs, seq -Thus we sort the rows accordingly before writing them. -""" - - -def usage(): - print( - "fetch_qm_faiures.py -k <apikey> -b <minimum build=20210329000000>" - "-d <days back=1> -l <last event time> -w <workdir=.>" - ) - print("") - print("Invokes the query 78691 and stores the result in a JSON file.") - print("-k <apikey>: Your personal telemetry API key (not the query key!).") - print("-d <daysback>: Number of days to go back. Default is 1.") - print("-b <minimum build>: The lowest build id we will fetch data for.") - print("-l <last event time>: Fetch only events after this. Default is 0.") - print("-w <workdir>: Working directory, default is '.'") - sys.exit(2) - - -days = 1 -lasteventtime = 0 -key = "undefined" -workdir = "." -minbuild = "20210329000000" - -try: - opts, args = getopt.getopt( - sys.argv[1:], - "k:b:d:l:w:", - ["key=", "build=", "days=", "lasteventtime=", "workdir="], - ) - for opt, arg in opts: - if opt == "-k": - key = arg - elif opt == "-d": - days = int(arg) - elif opt == "-l": - lasteventtime = int(arg) - elif opt == "-b": - minbuild = arg - elif opt == "-w": - workdir = arg -except getopt.GetoptError: - usage() - -if key == "undefined": - usage() - -start = utils.dateback(days) -year = start.year -month = start.month -day = start.day - -run = {} -lastrun = utils.getLastRunFromExecutionFile(workdir) -if "lasteventtime" in lastrun: - lasteventtime = lastrun["lasteventtime"] -run["workdir"] = workdir -run["daysback"] = days -run["minbuild"] = minbuild - -p_params = "p_year={:04d}&p_month={:02d}&p_day={:02d}&p_build={}" "&p_last={}".format( - year, month, day, minbuild, lasteventtime -) -print(p_params) -result = telemetry.query(key, 78691, p_params) -rows = result["query_result"]["data"]["rows"] -run["numrows"] = len(rows) -if run["numrows"] > 0: - lasteventtime = telemetry.getLastEventTimeAbs(rows) - run["lasteventtime"] = lasteventtime - rows.sort( - key=lambda row: "{}.{}.{}.{}.{:06d}".format( - row["client_id"], - row["session_id"], - row["seq"] >> 32, # thread_id - row["submit_timeabs"], - row["seq"] & 0x00000000FFFFFFFF, # seq, - ), - reverse=False, - ) - outfile = "{}/qmrows_until_{}.json".format(workdir, lasteventtime) - utils.writeJSONFile(outfile, rows) - run["rawfile"] = outfile -else: - print("No results found, maybe next time.") - run["lasteventtime"] = lasteventtime - -utils.addNewRunToExecutionFile(workdir, run) |