75 lines
2.1 KiB
Python
75 lines
2.1 KiB
Python
# Copyright 2020 The Chromium Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
"""Workaround for qemu-img bug on arm64 platforms with multiple cores.
|
|
|
|
Runs qemu-img command with timeout and retries the command if it hangs.
|
|
|
|
See:
|
|
crbug.com/1046861 QEMU is out of date; current version of qemu-img
|
|
is unstable
|
|
|
|
https://bugs.launchpad.net/qemu/+bug/1805256 qemu-img hangs on
|
|
rcu_call_ready_event logic in Aarch64 when converting images
|
|
|
|
TODO(crbug.com/1046861): Remove this workaround when the bug is fixed.
|
|
"""
|
|
|
|
import logging
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
|
|
|
|
# qemu-img p99 run time on Cavium ThunderX2 servers is 26 seconds.
|
|
# Using 2x the p99 time as the timeout.
|
|
QEMU_IMG_TIMEOUT_SEC = 52
|
|
|
|
|
|
def _ExecQemuImgWithTimeout(command):
|
|
"""Execute qemu-img command in subprocess with timeout.
|
|
|
|
Returns: None if command timed out or return code if command completed.
|
|
"""
|
|
|
|
logging.info('qemu-img starting')
|
|
command_output_file = tempfile.NamedTemporaryFile('w')
|
|
p = subprocess.Popen(command, stdout=command_output_file,
|
|
stderr=subprocess.STDOUT)
|
|
start_sec = time.time()
|
|
while p.poll() is None and time.time() - start_sec < QEMU_IMG_TIMEOUT_SEC:
|
|
time.sleep(1)
|
|
stop_sec = time.time()
|
|
logging.info('qemu-img duration: %f' % float(stop_sec - start_sec))
|
|
|
|
if p.poll() is None:
|
|
returncode = None
|
|
p.kill()
|
|
p.wait()
|
|
else:
|
|
returncode = p.returncode
|
|
|
|
log_level = logging.WARN if returncode else logging.DEBUG
|
|
for line in open(command_output_file.name, 'r'):
|
|
logging.log(log_level, 'qemu-img stdout: ' + line.strip())
|
|
|
|
return returncode
|
|
|
|
|
|
def ExecQemuImgWithRetry(command):
|
|
""" Execute qemu-img command in subprocess with 2 retries.
|
|
|
|
Raises CalledProcessError if command does not complete successfully.
|
|
"""
|
|
|
|
tries = 0
|
|
status = None
|
|
while status is None and tries <= 2:
|
|
tries += 1
|
|
status = _ExecQemuImgWithTimeout(command)
|
|
|
|
if status is None:
|
|
raise subprocess.CalledProcessError(-1, command)
|
|
if status:
|
|
raise subprocess.CalledProcessError(status, command)
|