diff options
Diffstat (limited to 'third_party/libwebrtc/tools_webrtc/perf/catapult_uploader.py')
| -rw-r--r-- | third_party/libwebrtc/tools_webrtc/perf/catapult_uploader.py | 310 |
1 files changed, 310 insertions, 0 deletions
diff --git a/third_party/libwebrtc/tools_webrtc/perf/catapult_uploader.py b/third_party/libwebrtc/tools_webrtc/perf/catapult_uploader.py new file mode 100644 index 0000000000..d07c287f28 --- /dev/null +++ b/third_party/libwebrtc/tools_webrtc/perf/catapult_uploader.py @@ -0,0 +1,310 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import datetime +import json +import subprocess +import time +import zlib + +from typing import Optional +import dataclasses +import httplib2 + +from tracing.value import histogram +from tracing.value import histogram_set +from tracing.value.diagnostics import generic_set +from tracing.value.diagnostics import reserved_infos + + +@dataclasses.dataclass +class UploaderOptions(): + """Required information to upload perf metrics. + + Attributes: + perf_dashboard_machine_group: The "master" the bots are grouped under. + This string is the group in the the perf dashboard path + group/bot/perf_id/metric/subtest. + bot: The bot running the test (e.g. webrtc-win-large-tests). + test_suite: The key for the test in the dashboard (i.e. what you select + in the top-level test suite selector in the dashboard + webrtc_git_hash: webrtc.googlesource.com commit hash. + commit_position: Commit pos corresponding to the git hash. + build_page_url: URL to the build page for this build. + dashboard_url: Which dashboard to use. + input_results_file: A HistogramSet proto file coming from WebRTC tests. + output_json_file: Where to write the output (for debugging). + wait_timeout_sec: Maximum amount of time in seconds that the script will + wait for the confirmation. + wait_polling_period_sec: Status will be requested from the Dashboard + every wait_polling_period_sec seconds. + """ + perf_dashboard_machine_group: str + bot: str + test_suite: str + webrtc_git_hash: str + commit_position: int + build_page_url: str + dashboard_url: str + input_results_file: str + output_json_file: Optional[str] = None + wait_timeout_sec: datetime.timedelta = datetime.timedelta(seconds=1200) + wait_polling_period_sec: datetime.timedelta = datetime.timedelta(seconds=120) + + +def _GenerateOauthToken(): + args = ['luci-auth', 'token'] + p = subprocess.Popen(args, + universal_newlines=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + if p.wait() == 0: + output = p.stdout.read() + return output.strip() + raise RuntimeError( + 'Error generating authentication token.\nStdout: %s\nStderr:%s' % + (p.stdout.read(), p.stderr.read())) + + +def _CreateHeaders(oauth_token): + return {'Authorization': 'Bearer %s' % oauth_token} + + +def _SendHistogramSet(url, histograms): + """Make a HTTP POST with the given JSON to the Performance Dashboard. + + Args: + url: URL of Performance Dashboard instance, e.g. + "https://chromeperf.appspot.com". + histograms: a histogram set object that contains the data to be sent. + """ + headers = _CreateHeaders(_GenerateOauthToken()) + + serialized = json.dumps(_ApplyHacks(histograms.AsDicts()), indent=4) + + if url.startswith('http://localhost'): + # The catapult server turns off compression in developer mode. + data = serialized + else: + data = zlib.compress(serialized.encode('utf-8')) + + print('Sending %d bytes to %s.' % (len(data), url + '/add_histograms')) + + http = httplib2.Http() + response, content = http.request(url + '/add_histograms', + method='POST', + body=data, + headers=headers) + return response, content + + +def _WaitForUploadConfirmation(url, upload_token, wait_timeout, + wait_polling_period): + """Make a HTTP GET requests to the Performance Dashboard untill upload + status is known or the time is out. + + Args: + url: URL of Performance Dashboard instance, e.g. + "https://chromeperf.appspot.com". + upload_token: String that identifies Performance Dashboard and can be used + for the status check. + wait_timeout: (datetime.timedelta) Maximum time to wait for the + confirmation. + wait_polling_period: (datetime.timedelta) Performance Dashboard will be + polled every wait_polling_period amount of time. + """ + assert wait_polling_period <= wait_timeout + + headers = _CreateHeaders(_GenerateOauthToken()) + http = httplib2.Http() + + oauth_refreshed = False + response = None + resp_json = None + current_time = datetime.datetime.now() + end_time = current_time + wait_timeout + next_poll_time = current_time + wait_polling_period + while datetime.datetime.now() < end_time: + current_time = datetime.datetime.now() + if next_poll_time > current_time: + time.sleep((next_poll_time - current_time).total_seconds()) + next_poll_time = datetime.datetime.now() + wait_polling_period + + response, content = http.request(url + '/uploads/' + upload_token, + method='GET', + headers=headers) + + print('Upload state polled. Response: %r.' % content) + + if not oauth_refreshed and response.status == 403: + print('Oauth token refreshed. Continue polling.') + headers = _CreateHeaders(_GenerateOauthToken()) + oauth_refreshed = True + continue + + if response.status != 200: + break + + resp_json = json.loads(content) + if resp_json['state'] == 'COMPLETED' or resp_json['state'] == 'FAILED': + break + + return response, resp_json + + +# Because of an issues on the Dashboard side few measurements over a large set +# can fail to upload. That would lead to the whole upload to be marked as +# failed. Check it, so it doesn't increase flakiness of our tests. +# TODO(crbug.com/1145904): Remove check after fixed. +def _CheckFullUploadInfo(url, upload_token, + min_measurements_amount=50, + max_failed_measurements_percent=0.03): + """Make a HTTP GET requests to the Performance Dashboard to get full info + about upload (including measurements). Checks if upload is correct despite + not having status "COMPLETED". + + Args: + url: URL of Performance Dashboard instance, e.g. + "https://chromeperf.appspot.com". + upload_token: String that identifies Performance Dashboard and can be used + for the status check. + min_measurements_amount: minimal amount of measurements that the upload + should have to start tolerating failures in particular measurements. + max_failed_measurements_percent: maximal percent of failured measurements + to tolerate. + """ + headers = _CreateHeaders(_GenerateOauthToken()) + http = httplib2.Http() + + response, content = http.request(url + '/uploads/' + upload_token + + '?additional_info=measurements', + method='GET', + headers=headers) + + if response.status != 200: + print('Failed to reach the dashboard to get full upload info.') + return False + + resp_json = json.loads(content) + print('Full upload info: %s.' % json.dumps(resp_json, indent=4)) + + if 'measurements' in resp_json: + measurements_cnt = len(resp_json['measurements']) + not_completed_state_cnt = len( + [m for m in resp_json['measurements'] if m['state'] != 'COMPLETED']) + + if (measurements_cnt >= min_measurements_amount + and (not_completed_state_cnt / + (measurements_cnt * 1.0) <= max_failed_measurements_percent)): + print(('Not all measurements were confirmed to upload. ' + 'Measurements count: %d, failed to upload or timed out: %d' % + (measurements_cnt, not_completed_state_cnt))) + return True + + return False + + +# TODO(https://crbug.com/1029452): HACKHACK +# Remove once we have doubles in the proto and handle -infinity correctly. +def _ApplyHacks(dicts): + def _NoInf(value): + if value == float('inf'): + return histogram.JS_MAX_VALUE + if value == float('-inf'): + return -histogram.JS_MAX_VALUE + return value + + for d in dicts: + if 'running' in d: + d['running'] = [_NoInf(value) for value in d['running']] + if 'sampleValues' in d: + d['sampleValues'] = [_NoInf(value) for value in d['sampleValues']] + + return dicts + + +def _LoadHistogramSetFromProto(options): + hs = histogram_set.HistogramSet() + with open(options.input_results_file, 'rb') as f: + hs.ImportProto(f.read()) + + return hs + + +def _AddBuildInfo(histograms, options): + common_diagnostics = { + reserved_infos.MASTERS: options.perf_dashboard_machine_group, + reserved_infos.BOTS: options.bot, + reserved_infos.POINT_ID: options.commit_position, + reserved_infos.BENCHMARKS: options.test_suite, + reserved_infos.WEBRTC_REVISIONS: str(options.webrtc_git_hash), + reserved_infos.BUILD_URLS: options.build_page_url, + } + + for k, v in list(common_diagnostics.items()): + histograms.AddSharedDiagnosticToAllHistograms(k.name, + generic_set.GenericSet([v])) + + +def _DumpOutput(histograms, output_file): + with open(output_file, 'w') as f: + json.dump(_ApplyHacks(histograms.AsDicts()), f, indent=4) + + +def UploadToDashboardImpl(options): + histograms = _LoadHistogramSetFromProto(options) + _AddBuildInfo(histograms, options) + + if options.output_json_file: + _DumpOutput(histograms, options.output_json_file) + + response, content = _SendHistogramSet(options.dashboard_url, histograms) + + if response.status != 200: + print(('Upload failed with %d: %s\n\n%s' % + (response.status, response.reason, content))) + return 1 + + upload_token = json.loads(content).get('token') + if not upload_token: + print(('Received 200 from dashboard. ', + 'Not waiting for the upload status confirmation.')) + return 0 + + response, resp_json = _WaitForUploadConfirmation( + options.dashboard_url, upload_token, options.wait_timeout_sec, + options.wait_polling_period_sec) + + if ((resp_json and resp_json['state'] == 'COMPLETED') + or _CheckFullUploadInfo(options.dashboard_url, upload_token)): + print('Upload completed.') + return 0 + + if response.status != 200: + print(('Upload status poll failed with %d: %s' % + (response.status, response.reason))) + return 1 + + if resp_json['state'] == 'FAILED': + print('Upload failed.') + return 1 + + print(('Upload wasn\'t completed in a given time: %s seconds.' % + options.wait_timeout_sec)) + return 1 + + +def UploadToDashboard(options): + try: + exit_code = UploadToDashboardImpl(options) + except RuntimeError as e: + print(e) + return 1 + return exit_code |