diff options
Diffstat (limited to 'src/VBox/ValidationKit/testdriver')
17 files changed, 20434 insertions, 0 deletions
diff --git a/src/VBox/ValidationKit/testdriver/Makefile.kmk b/src/VBox/ValidationKit/testdriver/Makefile.kmk new file mode 100644 index 00000000..4aa49adc --- /dev/null +++ b/src/VBox/ValidationKit/testdriver/Makefile.kmk @@ -0,0 +1,48 @@ +# $Id: Makefile.kmk $ +## @file +# VirtualBox Validation Kit - Python Test Driver. +# + +# +# Copyright (C) 2010-2022 Oracle and/or its affiliates. +# +# This file is part of VirtualBox base platform packages, as +# available from https://www.virtualbox.org. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation, in version 3 of the +# License. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see <https://www.gnu.org/licenses>. +# +# The contents of this file may alternatively be used under the terms +# of the Common Development and Distribution License Version 1.0 +# (CDDL), a copy of it is provided in the "COPYING.CDDL" file included +# in the VirtualBox distribution, in which case the provisions of the +# CDDL are applicable instead of those of the GPL. +# +# You may elect to license modified versions of this file under the +# terms and conditions of either the GPL or the CDDL or both. +# +# SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +# + +SUB_DEPTH = ../../../.. +include $(KBUILD_PATH)/subheader.kmk + + +VBOX_VALIDATIONKIT_PYTHON_SOURCES += $(filter-out %/winbase.py %/vboxcon.py, $(wildcard $(PATH_SUB_CURRENT)/*.py)) +ifeq ($(KBUILD_HOST),win) + VBOX_VALIDATIONKIT_PYTHON_SOURCES += $(PATH_SUB_CURRENT)/winbase.py +endif + +$(evalcall def_vbox_validationkit_process_python_sources) +include $(FILE_KBUILD_SUB_FOOTER) + diff --git a/src/VBox/ValidationKit/testdriver/__init__.py b/src/VBox/ValidationKit/testdriver/__init__.py new file mode 100644 index 00000000..f6534f5b --- /dev/null +++ b/src/VBox/ValidationKit/testdriver/__init__.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +# $Id: __init__.py $ + +""" +Test driver package +""" + +__copyright__ = \ +""" +Copyright (C) 2010-2022 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" + +__version__ = "$Revision: 153224 $"; + diff --git a/src/VBox/ValidationKit/testdriver/base.py b/src/VBox/ValidationKit/testdriver/base.py new file mode 100755 index 00000000..b90dd0a8 --- /dev/null +++ b/src/VBox/ValidationKit/testdriver/base.py @@ -0,0 +1,1860 @@ +# -*- coding: utf-8 -*- +# $Id: base.py $ +# pylint: disable=too-many-lines + +""" +Base testdriver module. +""" + +__copyright__ = \ +""" +Copyright (C) 2010-2022 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 154842 $" + + +# Standard Python imports. +import os +import os.path +import signal +import socket +import stat +import subprocess +import sys +import time +if sys.version_info[0] < 3: import thread; # pylint: disable=import-error +else: import _thread as thread; # pylint: disable=import-error +import threading +import traceback +import tempfile; +import unittest; + +# Validation Kit imports. +from common import utils; +from common.constants import rtexitcode; +from testdriver import reporter; +if sys.platform == 'win32': + from testdriver import winbase; + +# Figure where we are. +try: __file__ +except: __file__ = sys.argv[0]; +g_ksValidationKitDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))); + +# Python 3 hacks: +if sys.version_info[0] >= 3: + long = int; # pylint: disable=redefined-builtin,invalid-name + + +# +# Some utility functions. +# + +def exeSuff(): + """ + Returns the executable suffix. + """ + if os.name in ('nt', 'os2'): + return '.exe'; + return ''; + +def searchPath(sExecName): + """ + Searches the PATH for the specified executable name, returning the first + existing file/directory/whatever. The return is abspath'ed. + """ + sSuff = exeSuff(); + + sPath = os.getenv('PATH', os.getenv('Path', os.path.defpath)); + aPaths = sPath.split(os.path.pathsep) + for sDir in aPaths: + sFullExecName = os.path.join(sDir, sExecName); + if os.path.exists(sFullExecName): + return os.path.abspath(sFullExecName); + sFullExecName += sSuff; + if os.path.exists(sFullExecName): + return os.path.abspath(sFullExecName); + return sExecName; + +def getEnv(sVar, sLocalAlternative = None): + """ + Tries to get an environment variable, optionally with a local run alternative. + Will raise an exception if sLocalAlternative is None and the variable is + empty or missing. + """ + try: + sVal = os.environ.get(sVar, None); + if sVal is None: + raise GenError('environment variable "%s" is missing' % (sVar)); + if sVal == "": + raise GenError('environment variable "%s" is empty' % (sVar)); + except: + if sLocalAlternative is None or not reporter.isLocal(): + raise + sVal = sLocalAlternative; + return sVal; + +def getDirEnv(sVar, sAlternative = None, fLocalReq = False, fTryCreate = False): + """ + Tries to get an environment variable specifying a directory path. + + Resolves it into an absolute path and verifies its existance before + returning it. + + If the environment variable is empty or isn't set, or if the directory + doesn't exist or isn't a directory, sAlternative is returned instead. + If sAlternative is None, then we'll raise a GenError. For local runs we'll + only do this if fLocalReq is True. + """ + assert sAlternative is None or fTryCreate is False; + try: + sVal = os.environ.get(sVar, None); + if sVal is None: + raise GenError('environment variable "%s" is missing' % (sVar)); + if sVal == "": + raise GenError('environment variable "%s" is empty' % (sVar)); + + sVal = os.path.abspath(sVal); + if not os.path.isdir(sVal): + if not fTryCreate or os.path.exists(sVal): + reporter.error('the value of env.var. "%s" is not a dir: "%s"' % (sVar, sVal)); + raise GenError('the value of env.var. "%s" is not a dir: "%s"' % (sVar, sVal)); + try: + os.makedirs(sVal, 0o700); + except: + reporter.error('makedirs failed on the value of env.var. "%s": "%s"' % (sVar, sVal)); + raise GenError('makedirs failed on the value of env.var. "%s": "%s"' % (sVar, sVal)); + except: + if sAlternative is None: + if reporter.isLocal() and fLocalReq: + raise; + sVal = None; + else: + sVal = os.path.abspath(sAlternative); + return sVal; + +def timestampMilli(): + """ + Gets a millisecond timestamp. + """ + return utils.timestampMilli(); + +def timestampNano(): + """ + Gets a nanosecond timestamp. + """ + return utils.timestampNano(); + +def tryGetHostByName(sName): + """ + Wrapper around gethostbyname. + """ + if sName is not None: + try: + sIpAddr = socket.gethostbyname(sName); + except: + reporter.errorXcpt('gethostbyname(%s)' % (sName)); + else: + if sIpAddr != '0.0.0.0': + sName = sIpAddr; + else: + reporter.error('gethostbyname(%s) -> %s' % (sName, sIpAddr)); + return sName; + +def __processSudoKill(uPid, iSignal, fSudo): + """ + Does the sudo kill -signal pid thing if fSudo is true, else uses os.kill. + """ + try: + if fSudo: + return utils.sudoProcessCall(['/bin/kill', '-%s' % (iSignal,), str(uPid)]) == 0; + os.kill(uPid, iSignal); + return True; + except: + reporter.logXcpt('uPid=%s' % (uPid,)); + return False; + +def processInterrupt(uPid, fSudo = False): + """ + Sends a SIGINT or equivalent to interrupt the specified process. + Returns True on success, False on failure. + + On Windows hosts this may not work unless the process happens to be a + process group leader. + """ + if sys.platform == 'win32': + fRc = winbase.processInterrupt(uPid) + else: + fRc = __processSudoKill(uPid, signal.SIGINT, fSudo); + return fRc; + +def sendUserSignal1(uPid, fSudo = False): + """ + Sends a SIGUSR1 or equivalent to nudge the process into shutting down + (VBoxSVC) or something. + Returns True on success, False on failure or if not supported (win). + + On Windows hosts this may not work unless the process happens to be a + process group leader. + """ + if sys.platform == 'win32': + fRc = False; + else: + fRc = __processSudoKill(uPid, signal.SIGUSR1, fSudo); # pylint: disable=no-member + return fRc; + +def processTerminate(uPid, fSudo = False): + """ + Terminates the process in a nice manner (SIGTERM or equivalent). + Returns True on success, False on failure (logged). + """ + fRc = False; + if sys.platform == 'win32': + fRc = winbase.processTerminate(uPid); + else: + fRc = __processSudoKill(uPid, signal.SIGTERM, fSudo); + return fRc; + +def processKill(uPid, fSudo = False): + """ + Terminates the process with extreme prejudice (SIGKILL). + Returns True on success, False on failure. + """ + fRc = False; + if sys.platform == 'win32': + fRc = winbase.processKill(uPid); + else: + fRc = __processSudoKill(uPid, signal.SIGKILL, fSudo); # pylint: disable=no-member + return fRc; + +def processKillWithNameCheck(uPid, sName): + """ + Like processKill(), but checks if the process name matches before killing + it. This is intended for killing using potentially stale pid values. + + Returns True on success, False on failure. + """ + + if processCheckPidAndName(uPid, sName) is not True: + return False; + return processKill(uPid); + + +def processExists(uPid): + """ + Checks if the specified process exits. + This will only work if we can signal/open the process. + + Returns True if it positively exists, False otherwise. + """ + return utils.processExists(uPid); + +def processCheckPidAndName(uPid, sName): + """ + Checks if a process PID and NAME matches. + """ + if sys.platform == 'win32': + fRc = winbase.processCheckPidAndName(uPid, sName); + else: + sOs = utils.getHostOs(); + if sOs == 'linux': + asPsCmd = ['/bin/ps', '-p', '%u' % (uPid,), '-o', 'fname=']; + elif sOs == 'solaris': + asPsCmd = ['/usr/bin/ps', '-p', '%u' % (uPid,), '-o', 'fname=']; + elif sOs == 'darwin': + asPsCmd = ['/bin/ps', '-p', '%u' % (uPid,), '-o', 'ucomm=']; + else: + asPsCmd = None; + + if asPsCmd is not None: + try: + oPs = subprocess.Popen(asPsCmd, stdout=subprocess.PIPE); # pylint: disable=consider-using-with + sCurName = oPs.communicate()[0]; + iExitCode = oPs.wait(); + except: + reporter.logXcpt(); + return False; + + # ps fails with non-zero exit code if the pid wasn't found. + if iExitCode != 0: + return False; + if sCurName is None: + return False; + sCurName = sCurName.strip(); + if sCurName == '': + return False; + + if os.path.basename(sName) == sName: + sCurName = os.path.basename(sCurName); + elif os.path.basename(sCurName) == sCurName: + sName = os.path.basename(sName); + + if sCurName != sName: + return False; + + fRc = True; + return fRc; + +def wipeDirectory(sDir): + """ + Deletes all file and sub-directories in sDir, leaving sDir in empty afterwards. + Returns the number of errors after logging them as errors. + """ + if not os.path.exists(sDir): + return 0; + + try: + asNames = os.listdir(sDir); + except: + return reporter.errorXcpt('os.listdir("%s")' % (sDir)); + + cErrors = 0; + for sName in asNames: + # Build full path and lstat the object. + sFullName = os.path.join(sDir, sName) + try: + oStat = os.lstat(sFullName); + except: + reporter.errorXcpt('lstat("%s")' % (sFullName,)); + cErrors = cErrors + 1; + continue; + + if stat.S_ISDIR(oStat.st_mode): + # Directory - recurse and try remove it. + cErrors = cErrors + wipeDirectory(sFullName); + try: + os.rmdir(sFullName); + except: + reporter.errorXcpt('rmdir("%s")' % (sFullName,)); + cErrors = cErrors + 1; + else: + # File, symlink, fifo or something - remove/unlink. + try: + os.remove(sFullName); + except: + reporter.errorXcpt('remove("%s")' % (sFullName,)); + cErrors = cErrors + 1; + return cErrors; + + +# +# Classes +# + +class GenError(Exception): + """ + Exception class which only purpose it is to allow us to only catch our own + exceptions. Better design later. + """ + + def __init__(self, sWhat = "whatever"): + Exception.__init__(self); + self.sWhat = sWhat + + def str(self): + """Get the message string.""" + return self.sWhat; + + +class InvalidOption(GenError): + """ + Exception thrown by TestDriverBase.parseOption(). It contains the error message. + """ + def __init__(self, sWhat): + GenError.__init__(self, sWhat); + + +class QuietInvalidOption(GenError): + """ + Exception thrown by TestDriverBase.parseOption(). Error already printed, just + return failure. + """ + def __init__(self): + GenError.__init__(self, ""); + + +class TdTaskBase(object): + """ + The base task. + """ + + def __init__(self, sCaller, fnProcessEvents = None): + self.sDbgCreated = '%s: %s' % (utils.getTimePrefix(), sCaller); + self.fSignalled = False; + self.__oRLock = threading.RLock(); + self.oCv = threading.Condition(self.__oRLock); + self.oOwner = None; + self.msStart = timestampMilli(); + self.oLocker = None; + + ## Callback function that takes no parameters and will not be called holding the lock. + ## It is a hack to work the XPCOM and COM event queues, so we won't hold back events + ## that could block task progress (i.e. hangs VM). + self.fnProcessEvents = fnProcessEvents; + + def __del__(self): + """In case we need it later on.""" + pass; # pylint: disable=unnecessary-pass + + def toString(self): + """ + Stringifies the object, mostly as a debug aid. + """ + return '<%s: fSignalled=%s, __oRLock=%s, oCv=%s, oOwner=%s, oLocker=%s, msStart=%s, sDbgCreated=%s>' \ + % (type(self).__name__, self.fSignalled, self.__oRLock, self.oCv, repr(self.oOwner), self.oLocker, self.msStart, + self.sDbgCreated,); + + def __str__(self): + return self.toString(); + + def lockTask(self): + """ Wrapper around oCv.acquire(). """ + if True is True: # change to False for debugging deadlocks. # pylint: disable=comparison-with-itself + self.oCv.acquire(); + else: + msStartWait = timestampMilli(); + while self.oCv.acquire(0) is False: + if timestampMilli() - msStartWait > 30*1000: + reporter.error('!!! timed out waiting for %s' % (self, )); + traceback.print_stack(); + reporter.logAllStacks() + self.oCv.acquire(); + break; + time.sleep(0.5); + self.oLocker = thread.get_ident() + return None; + + def unlockTask(self): + """ Wrapper around oCv.release(). """ + self.oLocker = None; + self.oCv.release(); + return None; + + def getAgeAsMs(self): + """ + Returns the number of milliseconds the task has existed. + """ + return timestampMilli() - self.msStart; + + def setTaskOwner(self, oOwner): + """ + Sets or clears the task owner. (oOwner can be None.) + + Returns the previous owner, this means None if not owned. + """ + self.lockTask(); + oOldOwner = self.oOwner; + self.oOwner = oOwner; + self.unlockTask(); + return oOldOwner; + + def signalTaskLocked(self): + """ + Variant of signalTask that can be called while owning the lock. + """ + fOld = self.fSignalled; + if not fOld: + reporter.log2('signalTaskLocked(%s)' % (self,)); + self.fSignalled = True; + self.oCv.notifyAll(); # pylint: disable=deprecated-method + if self.oOwner is not None: + self.oOwner.notifyAboutReadyTask(self); + return fOld; + + def signalTask(self): + """ + Signals the task, internal use only. + + Returns the previous state. + """ + self.lockTask(); + fOld = self.signalTaskLocked(); + self.unlockTask(); + return fOld + + def resetTaskLocked(self): + """ + Variant of resetTask that can be called while owning the lock. + """ + fOld = self.fSignalled; + self.fSignalled = False; + return fOld; + + def resetTask(self): + """ + Resets the task signal, internal use only. + + Returns the previous state. + """ + self.lockTask(); + fOld = self.resetTaskLocked(); + self.unlockTask(); + return fOld + + def pollTask(self, fLocked = False): + """ + Poll the signal status of the task. + Returns True if signalled, False if not. + + Override this method. + """ + if not fLocked: + self.lockTask(); + fState = self.fSignalled; + if not fLocked: + self.unlockTask(); + return fState + + def waitForTask(self, cMsTimeout = 0): + """ + Waits for the task to be signalled. + + Returns True if the task is/became ready before the timeout expired. + Returns False if the task is still not after cMsTimeout have elapsed. + + Overriable. + """ + if self.fnProcessEvents: + self.fnProcessEvents(); + + self.lockTask(); + + fState = self.pollTask(True); + if not fState: + # Don't wait more than 1s. This allow lazy state polling and avoid event processing trouble. + msStart = timestampMilli(); + while not fState: + cMsElapsed = timestampMilli() - msStart; + if cMsElapsed >= cMsTimeout: + break; + + cMsWait = cMsTimeout - cMsElapsed + cMsWait = min(cMsWait, 1000); + try: + self.oCv.wait(cMsWait / 1000.0); + except: + pass; + + if self.fnProcessEvents: + self.unlockTask(); + self.fnProcessEvents(); + self.lockTask(); + + reporter.doPollWork('TdTaskBase.waitForTask'); + fState = self.pollTask(True); + + self.unlockTask(); + + if self.fnProcessEvents: + self.fnProcessEvents(); + + return fState; + + +class Process(TdTaskBase): + """ + Child Process. + """ + + def __init__(self, sName, asArgs, uPid, hWin = None, uTid = None): + TdTaskBase.__init__(self, utils.getCallerName()); + self.sName = sName; + self.asArgs = asArgs; + self.uExitCode = -127; + self.uPid = uPid; + self.hWin = hWin; + self.uTid = uTid; + self.sKindCrashReport = None; + self.sKindCrashDump = None; + + def toString(self): + return '<%s uExitcode=%s, uPid=%s, sName=%s, asArgs=%s, hWin=%s, uTid=%s>' \ + % (TdTaskBase.toString(self), self.uExitCode, self.uPid, self.sName, self.asArgs, self.hWin, self.uTid); + + # + # Instantiation methods. + # + + @staticmethod + def spawn(sName, *asArgsIn): + """ + Similar to os.spawnl(os.P_NOWAIT,). + + """ + # Make argument array (can probably use asArgsIn directly, but wtf). + asArgs = []; + for sArg in asArgsIn: + asArgs.append(sArg); + + # Special case: Windows. + if sys.platform == 'win32': + (uPid, hProcess, uTid) = winbase.processCreate(searchPath(sName), asArgs); + if uPid == -1: + return None; + return Process(sName, asArgs, uPid, hProcess, uTid); + + # Unixy. + try: + uPid = os.spawnv(os.P_NOWAIT, sName, asArgs); + except: + reporter.logXcpt('sName=%s' % (sName,)); + return None; + return Process(sName, asArgs, uPid); + + @staticmethod + def spawnp(sName, *asArgsIn): + """ + Similar to os.spawnlp(os.P_NOWAIT,). + + """ + return Process.spawn(searchPath(sName), *asArgsIn); + + # + # Task methods + # + + def pollTask(self, fLocked = False): + """ + Overridden pollTask method. + """ + if not fLocked: + self.lockTask(); + + fRc = self.fSignalled; + if not fRc: + if sys.platform == 'win32': + if winbase.processPollByHandle(self.hWin): + try: + if hasattr(self.hWin, '__int__'): # Needed for newer pywin32 versions. + (uPid, uStatus) = os.waitpid(self.hWin.__int__(), 0); + else: + (uPid, uStatus) = os.waitpid(self.hWin, 0); + if uPid in (self.hWin, self.uPid,): + self.hWin.Detach(); # waitpid closed it, so it's now invalid. + self.hWin = None; + uPid = self.uPid; + except: + reporter.logXcpt(); + uPid = self.uPid; + uStatus = 0xffffffff; + else: + uPid = 0; + uStatus = 0; # pylint: disable=redefined-variable-type + else: + try: + (uPid, uStatus) = os.waitpid(self.uPid, os.WNOHANG); # pylint: disable=no-member + except: + reporter.logXcpt(); + uPid = self.uPid; + uStatus = 0xffffffff; + + # Got anything? + if uPid == self.uPid: + self.uExitCode = uStatus; + reporter.log('Process %u -> %u (%#x)' % (uPid, uStatus, uStatus)); + self.signalTaskLocked(); + if self.uExitCode != 0 and (self.sKindCrashReport is not None or self.sKindCrashDump is not None): + reporter.error('Process "%s" returned/crashed with a non-zero status code!! rc=%u sig=%u%s (raw=%#x)' + % ( self.sName, self.uExitCode >> 8, self.uExitCode & 0x7f, + ' w/ core' if self.uExitCode & 0x80 else '', self.uExitCode)) + utils.processCollectCrashInfo(self.uPid, reporter.log, self._addCrashFile); + + fRc = self.fSignalled; + if not fLocked: + self.unlockTask(); + return fRc; + + def _addCrashFile(self, sFile, fBinary): + """ + Helper for adding a crash report or dump to the test report. + """ + sKind = self.sKindCrashDump if fBinary else self.sKindCrashReport; + if sKind is not None: + reporter.addLogFile(sFile, sKind); + return None; + + + # + # Methods + # + + def enableCrashReporting(self, sKindCrashReport, sKindCrashDump): + """ + Enabling (or disables) automatic crash reporting on systems where that + is possible. The two file kind parameters are on the form + 'crash/log/client' and 'crash/dump/client'. If both are None, + reporting will be disabled. + """ + self.sKindCrashReport = sKindCrashReport; + self.sKindCrashDump = sKindCrashDump; + + sCorePath = None; + sOs = utils.getHostOs(); + if sOs == 'solaris': + if sKindCrashDump is not None: # Enable. + sCorePath = getDirEnv('TESTBOX_PATH_SCRATCH', sAlternative = '/var/cores', fTryCreate = False); + (iExitCode, _, sErr) = utils.processOutputUnchecked([ 'coreadm', '-e', 'global', '-e', 'global-setid', \ + '-e', 'process', '-e', 'proc-setid', \ + '-g', os.path.join(sCorePath, '%f.%p.core')]); + else: # Disable. + (iExitCode, _, sErr) = utils.processOutputUnchecked([ 'coreadm', \ + '-d', 'global', '-d', 'global-setid', \ + '-d', 'process', '-d', 'proc-setid' ]); + if iExitCode != 0: # Don't report an actual error, just log this. + reporter.log('%s coreadm failed: %s' % ('Enabling' if sKindCrashDump else 'Disabling', sErr)); + + if sKindCrashDump is not None: + if sCorePath is not None: + reporter.log('Crash dumps enabled -- path is "%s"' % (sCorePath,)); + else: + reporter.log('Crash dumps disabled'); + + return True; + + def isRunning(self): + """ + Returns True if the process is still running, False if not. + """ + return not self.pollTask(); + + def wait(self, cMsTimeout = 0): + """ + Wait for the process to exit. + + Returns True if the process exited withint the specified wait period. + Returns False if still running. + """ + return self.waitForTask(cMsTimeout); + + def getExitCode(self): + """ + Returns the exit code of the process. + The process must have exited or the result will be wrong. + """ + if self.isRunning(): + return -127; + return self.uExitCode >> 8; + + def isNormalExit(self): + """ + Returns True if regular exit(), False if signal or still running. + """ + if self.isRunning(): + return False; + if sys.platform == 'win32': + return True; + return os.WIFEXITED(self.uExitCode); # pylint: disable=no-member + + def interrupt(self): + """ + Sends a SIGINT or equivalent to interrupt the process. + Returns True on success, False on failure. + + On Windows hosts this may not work unless the process happens to be a + process group leader. + """ + if sys.platform == 'win32': + return winbase.postThreadMesssageQuit(self.uTid); + return processInterrupt(self.uPid); + + def sendUserSignal1(self): + """ + Sends a SIGUSR1 or equivalent to nudge the process into shutting down + (VBoxSVC) or something. + Returns True on success, False on failure. + + On Windows hosts this may not work unless the process happens to be a + process group leader. + """ + #if sys.platform == 'win32': + # return winbase.postThreadMesssageClose(self.uTid); + return sendUserSignal1(self.uPid); + + def terminate(self): + """ + Terminates the process in a nice manner (SIGTERM or equivalent). + Returns True on success, False on failure (logged). + """ + if sys.platform == 'win32': + return winbase.processTerminateByHandle(self.hWin); + return processTerminate(self.uPid); + + def getPid(self): + """ Returns the process id. """ + return self.uPid; + + +class SubTestDriverBase(object): + """ + The base sub-test driver. + + It helps thinking of these as units/sets/groups of tests, where the test + cases are (mostly) realized in python. + + The sub-test drivers are subordinates of one or more test drivers. They + can be viewed as test code libraries that is responsible for parts of a + test driver run in different setups. One example would be testing a guest + additions component, which is applicable both to freshly installed guest + additions and VMs with old guest. + + The test drivers invokes the sub-test drivers in a private manner during + test execution, but some of the generic bits are done automagically by the + base class: options, help, resources, various other actions. + """ + + def __init__(self, oTstDrv, sName, sTestName): + self.oTstDrv = oTstDrv # type: TestDriverBase + self.sName = sName; # For use with options (--enable-sub-driver sName:sName2) + self.sTestName = sTestName; # More descriptive for passing to reporter.testStart(). + self.asRsrcs = [] # type: List(str) + self.fEnabled = True; # TestDriverBase --enable-sub-driver and --disable-sub-driver. + + def showUsage(self): + """ + Show usage information if any. + + The default implementation only prints the name. + """ + reporter.log(''); + reporter.log('Options for sub-test driver %s (%s):' % (self.sTestName, self.sName,)); + return True; + + def parseOption(self, asArgs, iArg): + """ + Parse an option. Override this. + + @param asArgs The argument vector. + @param iArg The index of the current argument. + + @returns The index of the next argument if consumed, @a iArg if not. + + @throws InvalidOption or QuietInvalidOption on syntax error or similar. + """ + _ = asArgs; + return iArg; + + +class TestDriverBase(object): # pylint: disable=too-many-instance-attributes + """ + The base test driver. + """ + + def __init__(self): + self.fInterrupted = False; + + # Actions. + self.asSpecialActions = ['extract', 'abort']; + self.asNormalActions = ['cleanup-before', 'verify', 'config', 'execute', 'cleanup-after' ]; + self.asActions = []; + self.sExtractDstPath = None; + + # Options. + self.fNoWipeClean = False; + + # Tasks - only accessed by one thread atm, so no need for locking. + self.aoTasks = []; + + # Host info. + self.sHost = utils.getHostOs(); + self.sHostArch = utils.getHostArch(); + + # Skipped status modifier (see end of innerMain()). + self.fBadTestbox = False; + + # + # Get our bearings and adjust the environment. + # + if not utils.isRunningFromCheckout(): + self.sBinPath = os.path.join(g_ksValidationKitDir, utils.getHostOs(), utils.getHostArch()); + else: + self.sBinPath = os.path.join(g_ksValidationKitDir, os.pardir, os.pardir, os.pardir, 'out', utils.getHostOsDotArch(), + os.environ.get('KBUILD_TYPE', 'debug'), + 'validationkit', utils.getHostOs(), utils.getHostArch()); + self.sOrgShell = os.environ.get('SHELL'); + self.sOurShell = os.path.join(self.sBinPath, 'vts_shell' + exeSuff()); # No shell yet. + os.environ['SHELL'] = self.sOurShell; + + self.sScriptPath = getDirEnv('TESTBOX_PATH_SCRIPTS'); + if self.sScriptPath is None: + self.sScriptPath = os.path.abspath(os.path.join(os.getcwd(), '..')); + os.environ['TESTBOX_PATH_SCRIPTS'] = self.sScriptPath; + + self.sScratchPath = getDirEnv('TESTBOX_PATH_SCRATCH', fTryCreate = True); + if self.sScratchPath is None: + sTmpDir = tempfile.gettempdir(); + if sTmpDir == '/tmp': # /var/tmp is generally more suitable on all platforms. + sTmpDir = '/var/tmp'; + self.sScratchPath = os.path.abspath(os.path.join(sTmpDir, 'VBoxTestTmp')); + if not os.path.isdir(self.sScratchPath): + os.makedirs(self.sScratchPath, 0o700); + os.environ['TESTBOX_PATH_SCRATCH'] = self.sScratchPath; + + self.sTestBoxName = getEnv( 'TESTBOX_NAME', 'local'); + self.sTestSetId = getEnv( 'TESTBOX_TEST_SET_ID', 'local'); + self.sBuildPath = getDirEnv('TESTBOX_PATH_BUILDS'); + self.sUploadPath = getDirEnv('TESTBOX_PATH_UPLOAD'); + self.sResourcePath = getDirEnv('TESTBOX_PATH_RESOURCES'); + if self.sResourcePath is None: + if self.sHost == 'darwin': self.sResourcePath = "/Volumes/testrsrc/"; + elif self.sHost == 'freebsd': self.sResourcePath = "/mnt/testrsrc/"; + elif self.sHost == 'linux': self.sResourcePath = "/mnt/testrsrc/"; + elif self.sHost == 'os2': self.sResourcePath = "T:/"; + elif self.sHost == 'solaris': self.sResourcePath = "/mnt/testrsrc/"; + elif self.sHost == 'win': self.sResourcePath = "T:/"; + else: raise GenError('unknown host OS "%s"' % (self.sHost)); + + # PID file for the testdriver. + self.sPidFile = os.path.join(self.sScratchPath, 'testdriver.pid'); + + # Some stuff for the log... + reporter.log('scratch: %s' % (self.sScratchPath,)); + + # Get the absolute timeout (seconds since epoch, see + # utils.timestampSecond()). None if not available. + self.secTimeoutAbs = os.environ.get('TESTBOX_TIMEOUT_ABS', None); + if self.secTimeoutAbs is not None: + self.secTimeoutAbs = long(self.secTimeoutAbs); + reporter.log('secTimeoutAbs: %s' % (self.secTimeoutAbs,)); + else: + reporter.log('TESTBOX_TIMEOUT_ABS not found in the environment'); + + # Distance from secTimeoutAbs that timeouts should be adjusted to. + self.secTimeoutFudge = 30; + + # List of sub-test drivers (SubTestDriverBase derivatives). + self.aoSubTstDrvs = [] # type: list(SubTestDriverBase) + + # Use the scratch path for temporary files. + if self.sHost in ['win', 'os2']: + os.environ['TMP'] = self.sScratchPath; + os.environ['TEMP'] = self.sScratchPath; + os.environ['TMPDIR'] = self.sScratchPath; + os.environ['IPRT_TMPDIR'] = self.sScratchPath; # IPRT/VBox specific. + + + # + # Resource utility methods. + # + + def isResourceFile(self, sFile): + """ + Checks if sFile is in in the resource set. + """ + ## @todo need to deal with stuff in the validationkit.zip and similar. + asRsrcs = self.getResourceSet(); + if sFile in asRsrcs: + return os.path.isfile(os.path.join(self.sResourcePath, sFile)); + for sRsrc in asRsrcs: + if sFile.startswith(sRsrc): + sFull = os.path.join(self.sResourcePath, sRsrc); + if os.path.isdir(sFull): + return os.path.isfile(os.path.join(self.sResourcePath, sRsrc)); + return False; + + def getFullResourceName(self, sName): + """ + Returns the full resource name. + """ + if os.path.isabs(sName): ## @todo Hack. Need to deal properly with stuff in the validationkit.zip and similar. + return sName; + return os.path.join(self.sResourcePath, sName); + + # + # Scratch related utility methods. + # + + def wipeScratch(self): + """ + Removes the content of the scratch directory. + Returns True on no errors, False + log entries on errors. + """ + cErrors = wipeDirectory(self.sScratchPath); + return cErrors == 0; + + # + # Sub-test driver related methods. + # + + def addSubTestDriver(self, oSubTstDrv): + """ + Adds a sub-test driver. + + Returns True on success, false on failure. + """ + assert isinstance(oSubTstDrv, SubTestDriverBase); + if oSubTstDrv in self.aoSubTstDrvs: + reporter.error('Attempt at adding sub-test driver %s twice.' % (oSubTstDrv.sName,)); + return False; + self.aoSubTstDrvs.append(oSubTstDrv); + return True; + + def showSubTstDrvUsage(self): + """ + Shows the usage of the sub-test drivers. + """ + for oSubTstDrv in self.aoSubTstDrvs: + oSubTstDrv.showUsage(); + return True; + + def subTstDrvParseOption(self, asArgs, iArgs): + """ + Lets the sub-test drivers have a go at the option. + Returns the index of the next option if handled, otherwise iArgs. + """ + for oSubTstDrv in self.aoSubTstDrvs: + iNext = oSubTstDrv.parseOption(asArgs, iArgs) + if iNext != iArgs: + assert iNext > iArgs; + assert iNext <= len(asArgs); + return iNext; + return iArgs; + + def findSubTstDrvByShortName(self, sShortName): + """ + Locates a sub-test driver by it's short name. + Returns sub-test driver object reference if found, None if not. + """ + for oSubTstDrv in self.aoSubTstDrvs: + if oSubTstDrv.sName == sShortName: + return oSubTstDrv; + return None; + + + # + # Task related methods. + # + + def addTask(self, oTask): + """ + Adds oTask to the task list. + + Returns True if the task was added. + + Returns False if the task was already in the task list. + """ + if oTask in self.aoTasks: + return False; + #reporter.log2('adding task %s' % (oTask,)); + self.aoTasks.append(oTask); + oTask.setTaskOwner(self); + #reporter.log2('tasks now in list: %d - %s' % (len(self.aoTasks), self.aoTasks)); + return True; + + def removeTask(self, oTask): + """ + Removes oTask to the task list. + + Returns oTask on success and None on failure. + """ + try: + #reporter.log2('removing task %s' % (oTask,)); + self.aoTasks.remove(oTask); + except: + return None; + else: + oTask.setTaskOwner(None); + #reporter.log2('tasks left: %d - %s' % (len(self.aoTasks), self.aoTasks)); + return oTask; + + def removeAllTasks(self): + """ + Removes all the task from the task list. + + Returns None. + """ + aoTasks = self.aoTasks; + self.aoTasks = []; + for oTask in aoTasks: + oTask.setTaskOwner(None); + return None; + + def notifyAboutReadyTask(self, oTask): + """ + Notificiation that there is a ready task. May be called owning the + task lock, so be careful wrt deadlocks. + + Remember to call super when overriding this. + """ + if oTask is None: pass; # lint + return None; + + def pollTasks(self): + """ + Polls the task to see if any of them are ready. + Returns the ready task, None if none are ready. + """ + for oTask in self.aoTasks: + if oTask.pollTask(): + return oTask; + return None; + + def waitForTasksSleepWorker(self, cMsTimeout): + """ + Overridable method that does the sleeping for waitForTask(). + + cMsTimeout will not be larger than 1000, so there is normally no need + to do any additional splitting up of the polling interval. + + Returns True if cMillieSecs elapsed. + Returns False if some exception was raised while we waited or + there turned out to be nothing to wait on. + """ + try: + self.aoTasks[0].waitForTask(cMsTimeout); + return True; + except Exception as oXcpt: + reporter.log("waitForTasksSleepWorker: %s" % (str(oXcpt),)); + return False; + + def waitForTasks(self, cMsTimeout): + """ + Waits for any of the tasks to require attention or a KeyboardInterrupt. + Returns the ready task on success, None on timeout or interrupt. + """ + try: + #reporter.log2('waitForTasks: cMsTimeout=%d' % (cMsTimeout,)); + + if cMsTimeout == 0: + return self.pollTasks(); + + if not self.aoTasks: + return None; + + fMore = True; + if cMsTimeout < 0: + while fMore: + oTask = self.pollTasks(); + if oTask is not None: + return oTask; + fMore = self.waitForTasksSleepWorker(1000); + else: + msStart = timestampMilli(); + while fMore: + oTask = self.pollTasks(); + if oTask is not None: + #reporter.log2('waitForTasks: returning %s, msStart=%d' % \ + # (oTask, msStart)); + return oTask; + + cMsElapsed = timestampMilli() - msStart; + if cMsElapsed > cMsTimeout: # not ==, we want the final waitForEvents. + break; + cMsSleep = cMsTimeout - cMsElapsed; + cMsSleep = min(cMsSleep, 1000); + fMore = self.waitForTasksSleepWorker(cMsSleep); + except KeyboardInterrupt: + self.fInterrupted = True; + reporter.errorXcpt('KeyboardInterrupt', 6); + except: + reporter.errorXcpt(None, 6); + return None; + + # + # PID file management methods. + # + + def pidFileRead(self): + """ + Worker that reads the PID file. + Returns dictionary of PID with value (sName, fSudo), empty if no file. + """ + dPids = {}; + if os.path.isfile(self.sPidFile): + try: + oFile = utils.openNoInherit(self.sPidFile, 'r'); + sContent = str(oFile.read()); + oFile.close(); + except: + reporter.errorXcpt(); + return dPids; + + sContent = str(sContent).strip().replace('\n', ' ').replace('\r', ' ').replace('\t', ' '); + for sProcess in sContent.split(' '): + asFields = sProcess.split(':'); + if len(asFields) == 3 and asFields[0].isdigit(): + try: + dPids[int(asFields[0])] = (asFields[2], asFields[1] == 'sudo'); + except: + reporter.logXcpt('sProcess=%s' % (sProcess,)); + else: + reporter.log('%s: "%s"' % (self.sPidFile, sProcess)); + + return dPids; + + def pidFileAdd(self, iPid, sName, fSudo = False): + """ + Adds a PID to the PID file, creating the file if necessary. + """ + try: + oFile = utils.openNoInherit(self.sPidFile, 'a'); + oFile.write('%s:%s:%s\n' + % ( iPid, + 'sudo' if fSudo else 'normal', + sName.replace(' ', '_').replace(':','_').replace('\n','_').replace('\r','_').replace('\t','_'),)); + oFile.close(); + except: + reporter.errorXcpt(); + return False; + ## @todo s/log/log2/ + reporter.log('pidFileAdd: added %s (%#x) %s fSudo=%s (new content: %s)' + % (iPid, iPid, sName, fSudo, self.pidFileRead(),)); + return True; + + def pidFileRemove(self, iPid, fQuiet = False): + """ + Removes a PID from the PID file. + """ + dPids = self.pidFileRead(); + if iPid not in dPids: + if not fQuiet: + reporter.log('pidFileRemove could not find %s in the PID file (content: %s)' % (iPid, dPids)); + return False; + + sName = dPids[iPid][0]; + del dPids[iPid]; + + sPid = ''; + for iPid2, tNameSudo in dPids.items(): + sPid += '%s:%s:%s\n' % (iPid2, 'sudo' if tNameSudo[1] else 'normal', tNameSudo[0]); + + try: + oFile = utils.openNoInherit(self.sPidFile, 'w'); + oFile.write(sPid); + oFile.close(); + except: + reporter.errorXcpt(); + return False; + ## @todo s/log/log2/ + reporter.log('pidFileRemove: removed PID %d [%s] (new content: %s)' % (iPid, sName, self.pidFileRead(),)); + return True; + + def pidFileDelete(self): + """Creates the testdriver PID file.""" + if os.path.isfile(self.sPidFile): + try: + os.unlink(self.sPidFile); + except: + reporter.logXcpt(); + return False; + ## @todo s/log/log2/ + reporter.log('pidFileDelete: deleted "%s"' % (self.sPidFile,)); + return True; + + # + # Misc helper methods. + # + + def requireMoreArgs(self, cMinNeeded, asArgs, iArg): + """ + Checks that asArgs has at least cMinNeeded args following iArg. + + Returns iArg + 1 if it checks out fine. + Raise appropritate exception if not, ASSUMING that the current argument + is found at iArg. + """ + assert cMinNeeded >= 1; + if iArg + cMinNeeded > len(asArgs): + if cMinNeeded > 1: + raise InvalidOption('The "%s" option takes %s values' % (asArgs[iArg], cMinNeeded,)); + raise InvalidOption('The "%s" option takes 1 value' % (asArgs[iArg],)); + return iArg + 1; + + def getBinTool(self, sName): + """ + Returns the full path to the given binary validation kit tool. + """ + return os.path.join(self.sBinPath, sName) + exeSuff(); + + def adjustTimeoutMs(self, cMsTimeout, cMsMinimum = None): + """ + Adjusts the given timeout (milliseconds) to take TESTBOX_TIMEOUT_ABS + and cMsMinimum (optional) into account. + + Returns adjusted timeout. + Raises no exceptions. + """ + if self.secTimeoutAbs is not None: + cMsToDeadline = self.secTimeoutAbs * 1000 - utils.timestampMilli(); + if cMsToDeadline >= 0: + # Adjust for fudge and enforce the minimum timeout + cMsToDeadline -= self.secTimeoutFudge * 1000; + if cMsToDeadline < (cMsMinimum if cMsMinimum is not None else 10000): + cMsToDeadline = cMsMinimum if cMsMinimum is not None else 10000; + + # Is the timeout beyond the (adjusted) deadline, if so change it. + if cMsTimeout > cMsToDeadline: + reporter.log('adjusting timeout: %s ms -> %s ms (deadline)\n' % (cMsTimeout, cMsToDeadline,)); + return cMsToDeadline; + reporter.log('adjustTimeoutMs: cMsTimeout (%s) > cMsToDeadline (%s)' % (cMsTimeout, cMsToDeadline,)); + else: + # Don't bother, we've passed the deadline. + reporter.log('adjustTimeoutMs: ooops! cMsToDeadline=%s (%s), timestampMilli()=%s, timestampSecond()=%s' + % (cMsToDeadline, cMsToDeadline*1000, utils.timestampMilli(), utils.timestampSecond())); + + # Only enforce the minimum timeout if specified. + if cMsMinimum is not None and cMsTimeout < cMsMinimum: + reporter.log('adjusting timeout: %s ms -> %s ms (minimum)\n' % (cMsTimeout, cMsMinimum,)); + cMsTimeout = cMsMinimum; + + return cMsTimeout; + + def prepareResultFile(self, sName = 'results.xml'): + """ + Given a base name (no path, but extension if required), a scratch file + name is computed and any previous file removed. + + Returns the full path to the file sName. + Raises exception on failure. + """ + sXmlFile = os.path.join(self.sScratchPath, sName); + if os.path.exists(sXmlFile): + os.unlink(sXmlFile); + return sXmlFile; + + + # + # Overridable methods. + # + + def showUsage(self): + """ + Shows the usage. + + When overriding this, call super first. + """ + sName = os.path.basename(sys.argv[0]); + reporter.log('Usage: %s [options] <action(s)>' % (sName,)); + reporter.log(''); + reporter.log('Actions (in execution order):'); + reporter.log(' cleanup-before'); + reporter.log(' Cleanups done at the start of testing.'); + reporter.log(' verify'); + reporter.log(' Verify that all necessary resources are present.'); + reporter.log(' config'); + reporter.log(' Configure the tests.'); + reporter.log(' execute'); + reporter.log(' Execute the tests.'); + reporter.log(' cleanup-after'); + reporter.log(' Cleanups done at the end of the testing.'); + reporter.log(''); + reporter.log('Special Actions:'); + reporter.log(' all'); + reporter.log(' Alias for: %s' % (' '.join(self.asNormalActions),)); + reporter.log(' extract <path>'); + reporter.log(' Extract the test resources and put them in the specified'); + reporter.log(' path for off side/line testing.'); + reporter.log(' abort'); + reporter.log(' Aborts the test.'); + reporter.log(''); + reporter.log('Base Options:'); + reporter.log(' -h, --help'); + reporter.log(' Show this help message.'); + reporter.log(' -v, --verbose'); + reporter.log(' Increase logging verbosity, repeat for more logging.'); + reporter.log(' -d, --debug'); + reporter.log(' Increase the debug logging level, repeat for more info.'); + reporter.log(' --no-wipe-clean'); + reporter.log(' Do not wipe clean the scratch area during the two clean up'); + reporter.log(' actions. This is for facilitating nested test driver execution.'); + if self.aoSubTstDrvs: + reporter.log(' --enable-sub-driver <sub1>[:..]'); + reporter.log(' --disable-sub-driver <sub1>[:..]'); + reporter.log(' Enables or disables one or more of the sub drivers: %s' + % (', '.join([oSubTstDrv.sName for oSubTstDrv in self.aoSubTstDrvs]),)); + return True; + + def parseOption(self, asArgs, iArg): + """ + Parse an option. Override this. + + Keyword arguments: + asArgs -- The argument vector. + iArg -- The index of the current argument. + + Returns iArg if the option was not recognized. + Returns the index of the next argument when something is consumed. + In the event of a syntax error, a InvalidOption or QuietInvalidOption + should be thrown. + """ + + if asArgs[iArg] in ('--help', '-help', '-h', '-?', '/?', '/help', '/H', '-H'): + self.showUsage(); + self.showSubTstDrvUsage(); + raise QuietInvalidOption(); + + # options + if asArgs[iArg] in ('--verbose', '-v'): + reporter.incVerbosity() + elif asArgs[iArg] in ('--debug', '-d'): + reporter.incDebug() + elif asArgs[iArg] == '--no-wipe-clean': + self.fNoWipeClean = True; + elif asArgs[iArg] in ('--enable-sub-driver', '--disable-sub-driver') and self.aoSubTstDrvs: + sOption = asArgs[iArg]; + iArg = self.requireMoreArgs(1, asArgs, iArg); + for sSubTstDrvName in asArgs[iArg].split(':'): + oSubTstDrv = self.findSubTstDrvByShortName(sSubTstDrvName); + if oSubTstDrv is None: + raise InvalidOption('Unknown sub-test driver given to %s: %s' % (sOption, sSubTstDrvName,)); + oSubTstDrv.fEnabled = sOption == '--enable-sub-driver'; + elif (asArgs[iArg] == 'all' or asArgs[iArg] in self.asNormalActions) \ + and self.asActions in self.asSpecialActions: + raise InvalidOption('selected special action "%s" already' % (self.asActions[0], )); + # actions + elif asArgs[iArg] == 'all': + self.asActions = [ 'all' ]; + elif asArgs[iArg] in self.asNormalActions: + self.asActions.append(asArgs[iArg]) + elif asArgs[iArg] in self.asSpecialActions: + if self.asActions: + raise InvalidOption('selected special action "%s" already' % (self.asActions[0], )); + self.asActions = [ asArgs[iArg] ]; + # extact <destination> + if asArgs[iArg] == 'extract': + iArg = iArg + 1; + if iArg >= len(asArgs): raise InvalidOption('The "extract" action requires a destination directory'); + self.sExtractDstPath = asArgs[iArg]; + else: + return iArg; + return iArg + 1; + + def completeOptions(self): + """ + This method is called after parsing all the options. + Returns success indicator. Use the reporter to complain. + + Overriable, call super. + """ + return True; + + def getResourceSet(self): + """ + Returns a set of file and/or directory names relative to + TESTBOX_PATH_RESOURCES. + + Override this, call super when using sub-test drivers. + """ + asRsrcs = []; + for oSubTstDrv in self.aoSubTstDrvs: + asRsrcs.extend(oSubTstDrv.asRsrcs); + return asRsrcs; + + def actionExtract(self): + """ + Handle the action that extracts the test resources for off site use. + Returns a success indicator and error details with the reporter. + + There is usually no need to override this. + """ + fRc = True; + asRsrcs = self.getResourceSet(); + for iRsrc, sRsrc in enumerate(asRsrcs): + reporter.log('Resource #%s: "%s"' % (iRsrc, sRsrc)); + sSrcPath = os.path.normpath(os.path.abspath(os.path.join(self.sResourcePath, sRsrc.replace('/', os.path.sep)))); + sDstPath = os.path.normpath(os.path.join(self.sExtractDstPath, sRsrc.replace('/', os.path.sep))); + + sDstDir = os.path.dirname(sDstPath); + if not os.path.exists(sDstDir): + try: os.makedirs(sDstDir, 0o775); + except: fRc = reporter.errorXcpt('Error creating directory "%s":' % (sDstDir,)); + + if os.path.isfile(sSrcPath): + try: utils.copyFileSimple(sSrcPath, sDstPath); + except: fRc = reporter.errorXcpt('Error copying "%s" to "%s":' % (sSrcPath, sDstPath,)); + elif os.path.isdir(sSrcPath): + fRc = reporter.error('Extracting directories have not been implemented yet'); + else: + fRc = reporter.error('Missing or unsupported resource type: %s' % (sSrcPath,)); + return fRc; + + def actionVerify(self): + """ + Handle the action that verify the test resources. + Returns a success indicator and error details with the reporter. + + There is usually no need to override this. + """ + + asRsrcs = self.getResourceSet(); + for sRsrc in asRsrcs: + # Go thru some pain to catch escape sequences. + if sRsrc.find("//") >= 0: + reporter.error('Double slash test resource name: "%s"' % (sRsrc)); + return False; + if sRsrc == ".." \ + or sRsrc.startswith("../") \ + or sRsrc.find("/../") >= 0 \ + or sRsrc.endswith("/.."): + reporter.error('Relative path in test resource name: "%s"' % (sRsrc)); + return False; + + sFull = os.path.normpath(os.path.abspath(os.path.join(self.sResourcePath, sRsrc))); + if not sFull.startswith(os.path.normpath(self.sResourcePath)): + reporter.error('sFull="%s" self.sResourcePath=%s' % (sFull, self.sResourcePath)); + reporter.error('The resource "%s" seems to specify a relative path' % (sRsrc)); + return False; + + reporter.log2('Checking for resource "%s" at "%s" ...' % (sRsrc, sFull)); + if os.path.isfile(sFull): + try: + oFile = utils.openNoInherit(sFull, "rb"); + oFile.close(); + except Exception as oXcpt: + reporter.error('The file resource "%s" cannot be accessed: %s' % (sFull, oXcpt)); + return False; + elif os.path.isdir(sFull): + if not os.path.isdir(os.path.join(sFull, '.')): + reporter.error('The directory resource "%s" cannot be accessed' % (sFull)); + return False; + elif os.path.exists(sFull): + reporter.error('The resource "%s" is not a file or directory' % (sFull)); + return False; + else: + reporter.error('The resource "%s" was not found' % (sFull)); + return False; + return True; + + def actionConfig(self): + """ + Handle the action that configures the test. + Returns True (success), False (failure) or None (skip the test), + posting complaints and explanations with the reporter. + + Override this. + """ + return True; + + def actionExecute(self): + """ + Handle the action that executes the test. + + Returns True (success), False (failure) or None (skip the test), + posting complaints and explanations with the reporter. + + Override this. + """ + return True; + + def actionCleanupBefore(self): + """ + Handle the action that cleans up spills from previous tests before + starting the tests. This is mostly about wiping the scratch space + clean in local runs. On a testbox the testbox script will use the + cleanup-after if the test is interrupted. + + Returns True (success), False (failure) or None (skip the test), + posting complaints and explanations with the reporter. + + Override this, but call super to wipe the scratch directory. + """ + if self.fNoWipeClean is False: + self.wipeScratch(); + return True; + + def actionCleanupAfter(self): + """ + Handle the action that cleans up all spills from executing the test. + + Returns True (success) or False (failure) posting complaints and + explanations with the reporter. + + Override this, but call super to wipe the scratch directory. + """ + if self.fNoWipeClean is False: + self.wipeScratch(); + return True; + + def actionAbort(self): + """ + Handle the action that aborts a (presumed) running testdriver, making + sure to include all it's children. + + Returns True (success) or False (failure) posting complaints and + explanations with the reporter. + + Override this, but call super to kill the testdriver script and any + other process covered by the testdriver PID file. + """ + + dPids = self.pidFileRead(); + reporter.log('The pid file contained: %s' % (dPids,)); + + # + # Try convince the processes to quit with increasing impoliteness. + # + if sys.platform == 'win32': + afnMethods = [ processInterrupt, processTerminate ]; + else: + afnMethods = [ sendUserSignal1, processInterrupt, processTerminate, processKill ]; + for fnMethod in afnMethods: + for iPid, tNameSudo in dPids.items(): + fnMethod(iPid, fSudo = tNameSudo[1]); + + for i in range(10): + if i > 0: + time.sleep(1); + + dPidsToRemove = []; # Temporary dict to append PIDs to remove later. + + for iPid, tNameSudo in dPids.items(): + if not processExists(iPid): + reporter.log('%s (%s) terminated' % (tNameSudo[0], iPid,)); + self.pidFileRemove(iPid, fQuiet = True); + dPidsToRemove.append(iPid); + continue; + + # Remove PIDs from original dictionary, as removing keys from a + # dictionary while iterating on it won't work and will result in a RuntimeError. + for iPidToRemove in dPidsToRemove: + del dPids[iPidToRemove]; + + if not dPids: + reporter.log('All done.'); + return True; + + if i in [4, 8]: + reporter.log('Still waiting for: %s (method=%s)' % (dPids, fnMethod,)); + + reporter.log('Failed to terminate the following processes: %s' % (dPids,)); + return False; + + + def onExit(self, iRc): + """ + Hook for doing very important cleanups on the way out. + + iRc is the exit code or -1 in the case of an unhandled exception. + Returns nothing and shouldn't raise exceptions (will be muted+ignored). + """ + _ = iRc; + return None; + + + # + # main() - don't override anything! + # + + def main(self, asArgs = None): + """ + The main function of the test driver. + + Keyword arguments: + asArgs -- The argument vector. Defaults to sys.argv. + + Returns exit code. No exceptions. + """ + + # + # Wrap worker in exception handler and always call a 'finally' like + # method to do crucial cleanups on the way out. + # + try: + iRc = self.innerMain(asArgs); + except: + reporter.logXcpt(cFrames = None); + try: + self.onExit(-1); + except: + reporter.logXcpt(); + raise; + self.onExit(iRc); + return iRc; + + + def innerMain(self, asArgs = None): # pylint: disable=too-many-statements + """ + Exception wrapped main() worker. + """ + + # + # Parse the arguments. + # + if asArgs is None: + asArgs = list(sys.argv); + iArg = 1; + try: + while iArg < len(asArgs): + iNext = self.parseOption(asArgs, iArg); + if iNext == iArg: + iNext = self.subTstDrvParseOption(asArgs, iArg); + if iNext == iArg: + raise InvalidOption('unknown option: %s' % (asArgs[iArg])) + iArg = iNext; + except QuietInvalidOption: + return rtexitcode.RTEXITCODE_SYNTAX; + except InvalidOption as oXcpt: + reporter.error(oXcpt.str()); + return rtexitcode.RTEXITCODE_SYNTAX; + except: + reporter.error('unexpected exception while parsing argument #%s' % (iArg)); + traceback.print_exc(); + return rtexitcode.RTEXITCODE_SYNTAX; + + if not self.completeOptions(): + return rtexitcode.RTEXITCODE_SYNTAX; + + if not self.asActions: + reporter.error('no action was specified'); + reporter.error('valid actions: %s' % (self.asNormalActions + self.asSpecialActions + ['all'])); + return rtexitcode.RTEXITCODE_SYNTAX; + + # + # Execte the actions. + # + fRc = True; # Tristate - True (success), False (failure), None (skipped). + asActions = list(self.asActions); # Must copy it or vboxinstaller.py breaks. + if 'extract' in asActions: + reporter.log('*** extract action ***'); + asActions.remove('extract'); + fRc = self.actionExtract(); + reporter.log('*** extract action completed (fRc=%s) ***' % (fRc)); + elif 'abort' in asActions: + reporter.appendToProcessName('/abort'); # Make it easier to spot in the log. + reporter.log('*** abort action ***'); + asActions.remove('abort'); + fRc = self.actionAbort(); + reporter.log('*** abort action completed (fRc=%s) ***' % (fRc)); + else: + if asActions == [ 'all' ]: + asActions = list(self.asNormalActions); + + if 'verify' in asActions: + reporter.log('*** verify action ***'); + asActions.remove('verify'); + fRc = self.actionVerify(); + if fRc is True: reporter.log("verified succeeded"); + else: reporter.log("verified failed (fRc=%s)" % (fRc,)); + reporter.log('*** verify action completed (fRc=%s) ***' % (fRc,)); + + if 'cleanup-before' in asActions: + reporter.log('*** cleanup-before action ***'); + asActions.remove('cleanup-before'); + fRc2 = self.actionCleanupBefore(); + if fRc2 is not True: reporter.log("cleanup-before failed"); + if fRc2 is not True and fRc is True: fRc = fRc2; + reporter.log('*** cleanup-before action completed (fRc2=%s, fRc=%s) ***' % (fRc2, fRc,)); + + self.pidFileAdd(os.getpid(), os.path.basename(sys.argv[0])); + + if 'config' in asActions and fRc is True: + asActions.remove('config'); + reporter.log('*** config action ***'); + fRc = self.actionConfig(); + if fRc is True: reporter.log("config succeeded"); + elif fRc is None: reporter.log("config skipping test"); + else: reporter.log("config failed"); + reporter.log('*** config action completed (fRc=%s) ***' % (fRc,)); + + if 'execute' in asActions and fRc is True: + asActions.remove('execute'); + reporter.log('*** execute action ***'); + fRc = self.actionExecute(); + if fRc is True: reporter.log("execute succeeded"); + elif fRc is None: reporter.log("execute skipping test"); + else: reporter.log("execute failed (fRc=%s)" % (fRc,)); + reporter.testCleanup(); + reporter.log('*** execute action completed (fRc=%s) ***' % (fRc,)); + + if 'cleanup-after' in asActions: + reporter.log('*** cleanup-after action ***'); + asActions.remove('cleanup-after'); + fRc2 = self.actionCleanupAfter(); + if fRc2 is not True: reporter.log("cleanup-after failed"); + if fRc2 is not True and fRc is True: fRc = fRc2; + reporter.log('*** cleanup-after action completed (fRc2=%s, fRc=%s) ***' % (fRc2, fRc,)); + + self.pidFileRemove(os.getpid()); + + if asActions and fRc is True: + reporter.error('unhandled actions: %s' % (asActions,)); + fRc = False; + + # + # Done - report the final result. + # + if fRc is None: + if self.fBadTestbox: + reporter.log('****************************************************************'); + reporter.log('*** The test driver SKIPPED the test because of BAD_TESTBOX. ***'); + reporter.log('****************************************************************'); + return rtexitcode.RTEXITCODE_BAD_TESTBOX; + reporter.log('*****************************************'); + reporter.log('*** The test driver SKIPPED the test. ***'); + reporter.log('*****************************************'); + return rtexitcode.RTEXITCODE_SKIPPED; + if fRc is not True: + reporter.error('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'); + reporter.error('!!! The test driver FAILED (in case we forgot to mention it). !!!'); + reporter.error('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'); + return rtexitcode.RTEXITCODE_FAILURE; + reporter.log('*******************************************'); + reporter.log('*** The test driver exits successfully. ***'); + reporter.log('*******************************************'); + return rtexitcode.RTEXITCODE_SUCCESS; + +# The old, deprecated name. +TestDriver = TestDriverBase; # pylint: disable=invalid-name + + +# +# Unit testing. +# + +# pylint: disable=missing-docstring +class TestDriverBaseTestCase(unittest.TestCase): + def setUp(self): + self.oTstDrv = TestDriverBase(); + self.oTstDrv.pidFileDelete(); + + def tearDown(self): + pass; # clean up scratch dir and such. + + def testPidFile(self): + + iPid1 = os.getpid() + 1; + iPid2 = os.getpid() + 2; + + self.assertTrue(self.oTstDrv.pidFileAdd(iPid1, 'test1')); + self.assertEqual(self.oTstDrv.pidFileRead(), {iPid1:('test1',False)}); + + self.assertTrue(self.oTstDrv.pidFileAdd(iPid2, 'test2', fSudo = True)); + self.assertEqual(self.oTstDrv.pidFileRead(), {iPid1:('test1',False), iPid2:('test2',True)}); + + self.assertTrue(self.oTstDrv.pidFileRemove(iPid1)); + self.assertEqual(self.oTstDrv.pidFileRead(), {iPid2:('test2',True)}); + + self.assertTrue(self.oTstDrv.pidFileRemove(iPid2)); + self.assertEqual(self.oTstDrv.pidFileRead(), {}); + + self.assertTrue(self.oTstDrv.pidFileDelete()); + +if __name__ == '__main__': + unittest.main(); + # not reached. diff --git a/src/VBox/ValidationKit/testdriver/btresolver.py b/src/VBox/ValidationKit/testdriver/btresolver.py new file mode 100755 index 00000000..b89d860c --- /dev/null +++ b/src/VBox/ValidationKit/testdriver/btresolver.py @@ -0,0 +1,626 @@ +# -*- coding: utf-8 -*- +# $Id: btresolver.py $ +# pylint: disable=too-many-lines + +""" +Backtrace resolver using external debugging symbols and RTLdrFlt. +""" + +__copyright__ = \ +""" +Copyright (C) 2016-2022 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 154739 $" + + +# Standard Python imports. +import os; +import re; +import shutil; +import subprocess; + +# Validation Kit imports. +from common import utils; + +def getRTLdrFltPath(asPaths): + """ + Returns the path to the RTLdrFlt tool looking in the provided paths + or None if not found. + """ + + for sPath in asPaths: + for sDirPath, _, asFiles in os.walk(sPath): + if 'RTLdrFlt' in asFiles: + return os.path.join(sDirPath, 'RTLdrFlt'); + + return None; + + + +class BacktraceResolverOs(object): + """ + Base class for all OS specific resolvers. + """ + + def __init__(self, sScratchPath, sBuildRoot, fnLog = None): + self.sScratchPath = sScratchPath; + self.sBuildRoot = sBuildRoot; + self.fnLog = fnLog; + + def log(self, sText): + """ + Internal logger callback. + """ + if self.fnLog is not None: + self.fnLog(sText); + + + +class BacktraceResolverOsLinux(BacktraceResolverOs): + """ + Linux specific backtrace resolver. + """ + + def __init__(self, sScratchPath, sBuildRoot, fnLog = None): + """ + Constructs a Linux host specific backtrace resolver. + """ + BacktraceResolverOs.__init__(self, sScratchPath, sBuildRoot, fnLog); + + self.asDbgFiles = {}; + + def prepareEnv(self): + """ + Prepares the environment for annotating Linux reports. + """ + fRc = False; + try: + sDbgArchive = os.path.join(self.sBuildRoot, 'bin', 'VirtualBox-dbg.tar.bz2'); + + # Extract debug symbol archive if it was found. + if os.path.exists(sDbgArchive): + asMembers = utils.unpackFile(sDbgArchive, self.sScratchPath, self.fnLog, + self.fnLog); + if asMembers: + # Populate the list of debug files. + for sMember in asMembers: + if os.path.isfile(sMember): + self.asDbgFiles[os.path.basename(sMember)] = sMember; + fRc = True; + except: + self.log('Failed to setup debug symbols'); + + return fRc; + + def cleanupEnv(self): + """ + Cleans up the environment. + """ + fRc = False; + try: + shutil.rmtree(self.sScratchPath, True); + fRc = True; + except: + pass; + + return fRc; + + def getDbgSymPathFromBinary(self, sBinary, sArch): + """ + Returns the path to file containing the debug symbols for the specified binary. + """ + _ = sArch; + sDbgFilePath = None; + try: + sDbgFilePath = self.asDbgFiles[sBinary]; + except: + pass; + + return sDbgFilePath; + + def getBinaryListWithLoadAddrFromReport(self, asReport): + """ + Parses the given VM state report and returns a list of binaries and their + load address. + + Returns a list if tuples containing the binary and load addres or an empty + list on failure. + """ + asListBinaries = []; + + # Look for the line "Mapped address spaces:" + iLine = 0; + while iLine < len(asReport): + if asReport[iLine].startswith('Mapped address spaces:'): + break; + iLine += 1; + + for sLine in asReport[iLine:]: + asCandidate = sLine.split(); + if len(asCandidate) == 5 \ + and asCandidate[0].startswith('0x') \ + and asCandidate[1].startswith('0x') \ + and asCandidate[2].startswith('0x') \ + and (asCandidate[3] == '0x0' or asCandidate[3] == '0')\ + and 'VirtualBox' in asCandidate[4]: + asListBinaries.append((asCandidate[0], os.path.basename(asCandidate[4]))); + + return asListBinaries; + + + +class BacktraceResolverOsDarwin(BacktraceResolverOs): + """ + Darwin specific backtrace resolver. + """ + + def __init__(self, sScratchPath, sBuildRoot, fnLog = None): + """ + Constructs a Linux host specific backtrace resolver. + """ + BacktraceResolverOs.__init__(self, sScratchPath, sBuildRoot, fnLog); + + self.asDbgFiles = {}; + + def prepareEnv(self): + """ + Prepares the environment for annotating Darwin reports. + """ + fRc = False; + try: + # + # Walk the scratch path directory and look for .dSYM directories, building a + # list of them. + # + asDSymPaths = []; + + for sDirPath, asDirs, _ in os.walk(self.sBuildRoot): + for sDir in asDirs: + if sDir.endswith('.dSYM'): + asDSymPaths.append(os.path.join(sDirPath, sDir)); + + # Expand the dSYM paths to full DWARF debug files in the next step + # and add them to the debug files dictionary. + for sDSymPath in asDSymPaths: + sBinary = os.path.basename(sDSymPath).strip('.dSYM'); + self.asDbgFiles[sBinary] = os.path.join(sDSymPath, 'Contents', 'Resources', + 'DWARF', sBinary); + + fRc = True; + except: + self.log('Failed to setup debug symbols'); + + return fRc; + + def cleanupEnv(self): + """ + Cleans up the environment. + """ + fRc = False; + try: + shutil.rmtree(self.sScratchPath, True); + fRc = True; + except: + pass; + + return fRc; + + def getDbgSymPathFromBinary(self, sBinary, sArch): + """ + Returns the path to file containing the debug symbols for the specified binary. + """ + # Hack to exclude executables as RTLdrFlt has some problems with it currently. + _ = sArch; + sDbgSym = None; + try: + sDbgSym = self.asDbgFiles[sBinary]; + except: + pass; + + if sDbgSym is not None and sDbgSym.endswith('.dylib'): + return sDbgSym; + + return None; + + def _getReportVersion(self, asReport): + """ + Returns the version of the darwin report. + """ + # Find the line starting with "Report Version:" + iLine = 0; + iVersion = 0; + while iLine < len(asReport): + if asReport[iLine].startswith('Report Version:'): + break; + iLine += 1; + + if iLine < len(asReport): + # Look for the start of the number + sVersion = asReport[iLine]; + iStartVersion = len('Report Version:'); + iEndVersion = len(sVersion); + + while iStartVersion < len(sVersion) \ + and not sVersion[iStartVersion:iStartVersion+1].isdigit(): + iStartVersion += 1; + + while iEndVersion > 0 \ + and not sVersion[iEndVersion-1:iEndVersion].isdigit(): + iEndVersion -= 1; + + iVersion = int(sVersion[iStartVersion:iEndVersion]); + else: + self.log('Couldn\'t find the report version'); + + return iVersion; + + def _getListOfBinariesFromReportPreSierra(self, asReport): + """ + Returns a list of loaded binaries with their load address obtained from + a pre Sierra report. + """ + asListBinaries = []; + + # Find the line starting with "Binary Images:" + iLine = 0; + while iLine < len(asReport): + if asReport[iLine].startswith('Binary Images:'): + break; + iLine += 1; + + if iLine < len(asReport): + # List starts after that + iLine += 1; + + # A line for a loaded binary looks like the following: + # 0x100042000 - 0x100095fff +VBoxDDU.dylib (4.3.15) <EB19C44D-F882-0803-DBDD-9995723111B7> /Application... + # We need the start address and the library name. + # To distinguish between our own libraries and ones from Apple we check whether the path at the end starts with + # /Applications/VirtualBox.app/Contents/MacOS + oRegExpPath = re.compile(r'/VirtualBox.app/Contents/MacOS'); + oRegExpAddr = re.compile(r'0x\w+'); + oRegExpBinPath = re.compile(r'VirtualBox.app/Contents/MacOS/\S*'); + while iLine < len(asReport): + asMatches = oRegExpPath.findall(asReport[iLine]); + if asMatches: + # Line contains the path, extract start address and path to binary + sAddr = oRegExpAddr.findall(asReport[iLine]); + sPath = oRegExpBinPath.findall(asReport[iLine]); + + if sAddr and sPath: + # Construct the path in into the build cache containing the debug symbols + oRegExp = re.compile(r'\w+\.{0,1}\w*$'); + sFilename = oRegExp.findall(sPath[0]); + + asListBinaries.append((sAddr[0], sFilename[0])); + else: + break; # End of image list + iLine += 1; + else: + self.log('Couldn\'t find the list of loaded binaries in the given report'); + + return asListBinaries; + + def _getListOfBinariesFromReportSierra(self, asReport): + """ + Returns a list of loaded binaries with their load address obtained from + a Sierra+ report. + """ + asListBinaries = []; + + # A line for a loaded binary looks like the following: + # 4 VBoxXPCOMIPCC.dylib 0x00000001139f17ea 0x1139e4000 + 55274 + # We need the start address and the library name. + # To distinguish between our own libraries and ones from Apple we check whether the library + # name contains VBox or VirtualBox + iLine = 0; + while iLine < len(asReport): + asStackTrace = asReport[iLine].split(); + + # Check whether the line is made up of 6 elements separated by whitespace + # and the first one is a number. + if len(asStackTrace) == 6 and asStackTrace[0].isdigit() \ + and (asStackTrace[1].find('VBox') != -1 or asStackTrace[1].find('VirtualBox') != -1) \ + and asStackTrace[3].startswith('0x'): + + # Check whether the library is already in our list an only add new ones + fFound = False; + for _, sLibrary in asListBinaries: + if asStackTrace[1] == sLibrary: + fFound = True; + break; + + if not fFound: + asListBinaries.append((asStackTrace[3], asStackTrace[1])); + iLine += 1; + + return asListBinaries; + + def getBinaryListWithLoadAddrFromReport(self, asReport): + """ + Parses the given VM state report and returns a list of binaries and their + load address. + + Returns a list if tuples containing the binary and load addres or an empty + list on failure. + """ + asListBinaries = []; + + iVersion = self._getReportVersion(asReport); + if iVersion > 0: + if iVersion <= 11: + self.log('Pre Sierra Report'); + asListBinaries = self._getListOfBinariesFromReportPreSierra(asReport); + elif iVersion == 12: + self.log('Sierra report'); + asListBinaries = self._getListOfBinariesFromReportSierra(asReport); + else: + self.log('Unsupported report version %s' % (iVersion, )); + + return asListBinaries; + + + +class BacktraceResolverOsSolaris(BacktraceResolverOs): + """ + Solaris specific backtrace resolver. + """ + + def __init__(self, sScratchPath, sBuildRoot, fnLog = None): + """ + Constructs a Linux host specific backtrace resolver. + """ + BacktraceResolverOs.__init__(self, sScratchPath, sBuildRoot, fnLog); + + self.asDbgFiles = {}; + + def prepareEnv(self): + """ + Prepares the environment for annotating Linux reports. + """ + fRc = False; + try: + sDbgArchive = os.path.join(self.sBuildRoot, 'bin', 'VirtualBoxDebug.tar.bz2'); + + # Extract debug symbol archive if it was found. + if os.path.exists(sDbgArchive): + asMembers = utils.unpackFile(sDbgArchive, self.sScratchPath, self.fnLog, + self.fnLog); + if asMembers: + # Populate the list of debug files. + for sMember in asMembers: + if os.path.isfile(sMember): + sArch = ''; + if 'amd64' in sMember: + sArch = 'amd64'; + else: + sArch = 'x86'; + self.asDbgFiles[os.path.basename(sMember) + '/' + sArch] = sMember; + fRc = True; + else: + self.log('Unpacking the debug archive failed'); + except: + self.log('Failed to setup debug symbols'); + + return fRc; + + def cleanupEnv(self): + """ + Cleans up the environment. + """ + fRc = False; + try: + shutil.rmtree(self.sScratchPath, True); + fRc = True; + except: + pass; + + return fRc; + + def getDbgSymPathFromBinary(self, sBinary, sArch): + """ + Returns the path to file containing the debug symbols for the specified binary. + """ + sDbgFilePath = None; + try: + sDbgFilePath = self.asDbgFiles[sBinary + '/' + sArch]; + except: + pass; + + return sDbgFilePath; + + def getBinaryListWithLoadAddrFromReport(self, asReport): + """ + Parses the given VM state report and returns a list of binaries and their + load address. + + Returns a list if tuples containing the binary and load addres or an empty + list on failure. + """ + asListBinaries = []; + + # Look for the beginning of the process address space mappings" + for sLine in asReport: + asItems = sLine.split(); + if len(asItems) == 4 \ + and asItems[3].startswith('/opt/VirtualBox') \ + and ( asItems[2] == 'r-x--' \ + or asItems[2] == 'r-x----'): + fFound = False; + sBinaryFile = os.path.basename(asItems[3]); + for _, sBinary in asListBinaries: + if sBinary == sBinaryFile: + fFound = True; + break; + if not fFound: + asListBinaries.append(('0x' + asItems[0], sBinaryFile)); + + return asListBinaries; + + + +class BacktraceResolver(object): + """ + A backtrace resolving class. + """ + + def __init__(self, sScratchPath, sBuildRoot, sTargetOs, sArch, sRTLdrFltPath = None, fnLog = None): + """ + Constructs a backtrace resolver object for the given target OS, + architecture and path to the directory containing the debug symbols and tools + we need. + """ + # Initialize all members first. + self.sScratchPath = sScratchPath; + self.sBuildRoot = sBuildRoot; + self.sTargetOs = sTargetOs; + self.sArch = sArch; + self.sRTLdrFltPath = sRTLdrFltPath; + self.fnLog = fnLog; + self.sDbgSymPath = None; + self.oResolverOs = None; + self.sScratchDbgPath = os.path.join(self.sScratchPath, 'dbgsymbols'); + + if self.fnLog is None: + self.fnLog = self.logStub; + + if self.sRTLdrFltPath is None: + self.sRTLdrFltPath = getRTLdrFltPath([self.sScratchPath, self.sBuildRoot]); + if self.sRTLdrFltPath is not None: + self.log('Found RTLdrFlt in %s' % (self.sRTLdrFltPath,)); + else: + self.log('Couldn\'t find RTLdrFlt in either %s or %s' % (self.sScratchPath, self.sBuildRoot)); + + def log(self, sText): + """ + Internal logger callback. + """ + if self.fnLog is not None: + self.fnLog(sText); + + def logStub(self, sText): + """ + Logging stub doing nothing. + """ + _ = sText; + + def prepareEnv(self): + """ + Prepares the environment to annotate backtraces, finding the required tools + and retrieving the debug symbols depending on the host OS. + + Returns True on success and False on error or if not supported. + """ + + # No access to the RTLdrFlt tool means no symbols so no point in trying + # to set something up. + if self.sRTLdrFltPath is None: + return False; + + # Create a directory containing the scratch space for the OS resolver backends. + fRc = True; + if not os.path.exists(self.sScratchDbgPath): + try: + os.makedirs(self.sScratchDbgPath, 0o750); + except: + fRc = False; + self.log('Failed to create scratch directory for debug symbols'); + + if fRc: + if self.sTargetOs == 'linux': + self.oResolverOs = BacktraceResolverOsLinux(self.sScratchDbgPath, self.sScratchPath, self.fnLog); + elif self.sTargetOs == 'darwin': + self.oResolverOs = BacktraceResolverOsDarwin(self.sScratchDbgPath, self.sScratchPath, self.fnLog); # pylint: disable=redefined-variable-type + elif self.sTargetOs == 'solaris': + self.oResolverOs = BacktraceResolverOsSolaris(self.sScratchDbgPath, self.sScratchPath, self.fnLog); # pylint: disable=redefined-variable-type + else: + self.log('The backtrace resolver is not supported on %s' % (self.sTargetOs,)); + fRc = False; + + if fRc: + fRc = self.oResolverOs.prepareEnv(); + if not fRc: + self.oResolverOs = None; + + if not fRc: + shutil.rmtree(self.sScratchDbgPath, True) + + return fRc; + + def cleanupEnv(self): + """ + Prepares the environment to annotate backtraces, finding the required tools + and retrieving the debug symbols depending on the host OS. + + Returns True on success and False on error or if not supported. + """ + fRc = False; + if self.oResolverOs is not None: + fRc = self.oResolverOs.cleanupEnv(); + + shutil.rmtree(self.sScratchDbgPath, True); + return fRc; + + def annotateReport(self, sReport): + """ + Annotates the given report with the previously prepared environment. + + Returns the annotated report on success or None on failure. + """ + sReportAn = None; + + if self.oResolverOs is not None: + asListBinaries = self.oResolverOs.getBinaryListWithLoadAddrFromReport(sReport.split('\n')); + + if asListBinaries: + asArgs = [self.sRTLdrFltPath, ]; + + for sLoadAddr, sBinary in asListBinaries: + sDbgSymPath = self.oResolverOs.getDbgSymPathFromBinary(sBinary, self.sArch); + if sDbgSymPath is not None: + asArgs.append(sDbgSymPath); + asArgs.append(sLoadAddr); + + oRTLdrFltProc = subprocess.Popen(asArgs, stdin=subprocess.PIPE, # pylint: disable=consider-using-with + stdout=subprocess.PIPE, bufsize=0); + if oRTLdrFltProc is not None: + try: + sReportAn, _ = oRTLdrFltProc.communicate(sReport); + except: + self.log('Retrieving annotation report failed (broken pipe / no matching interpreter?)'); + else: + self.log('Error spawning RTLdrFlt process'); + else: + self.log('Getting list of loaded binaries failed'); + else: + self.log('Backtrace resolver not fully initialized, not possible to annotate'); + + return sReportAn; + diff --git a/src/VBox/ValidationKit/testdriver/reporter.py b/src/VBox/ValidationKit/testdriver/reporter.py new file mode 100755 index 00000000..aad672d4 --- /dev/null +++ b/src/VBox/ValidationKit/testdriver/reporter.py @@ -0,0 +1,1984 @@ +# -*- coding: utf-8 -*- +# $Id: reporter.py $ +# pylint: disable=too-many-lines + +""" +Testdriver reporter module. +""" + +from __future__ import print_function; + +__copyright__ = \ +""" +Copyright (C) 2010-2022 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 153426 $" + + +# Standard Python imports. +import array +import datetime +import errno +import gc +import os +import os.path +import sys +import time +import threading +import traceback + +# Validation Kit imports. +from common import utils; + +## test reporter instance +g_oReporter = None # type: ReporterBase +g_sReporterName = None; + + +class ReporterLock(object): + """ + Work around problem with garbage collection triggering __del__ method with + logging while inside the logger lock and causing a deadlock. + """ + + def __init__(self, sName): + self.sName = sName; + self.oLock = threading.RLock(); + self.oOwner = None; + self.cRecursion = 0; + self.fRestoreGC = False; + + def acquire(self): + """ Acquire the lock. """ + oSelf = threading.current_thread(); + + # Take the lock. + if not self.oLock.acquire(): # pylint: disable=consider-using-with + return False; + + self.oOwner = oSelf; + self.cRecursion += 1; + + # Disable GC to avoid __del__ w/ log statement randomly reenter the logger. + if self.cRecursion == 1: + self.fRestoreGC = gc.isenabled(); + if self.fRestoreGC: + gc.disable(); + + return True; + + def release(self): + """ Release the lock. """ + oSelf = threading.current_thread(); + + # Check the ownership. + if oSelf != self.oOwner: + raise threading.ThreadError(); + + # Drop one recursion. + self.cRecursion -= 1; + if self.cRecursion <= 0: + + # Final recursion. Clear owner and re-enable GC. + self.oOwner = None; + if self.fRestoreGC: + self.fRestoreGC = False; + gc.enable(); + + self.oLock.release(); + +## Reporter lock. +g_oLock = ReporterLock('reporter'); + + + +class PythonLoggingStream(object): + """ + Python logging => testdriver/reporter.py stream. + """ + + def write(self, sText): + """Writes python log message to our stream.""" + if g_oReporter is not None: + sText = sText.rstrip("\r\n"); + #g_oReporter.log(0, 'python: %s' % (sText), utils.getCallerName(), utils.getTimePrefix()); + return True; + + def flush(self): + """Flushes the stream.""" + return True; + + +class ReporterBase(object): + """ + Base class for the reporters. + """ + + def __init__(self): + self.iVerbose = 1; + self.iDebug = 0; + self.cErrors = 0; + self.fTimedOut = False; # Once set, it trickles all the way up. + self.atTests = []; + self.sName = os.path.splitext(os.path.basename(sys.argv[0]))[0]; + + # Hook into the python logging. + import logging; + logging.basicConfig(stream = PythonLoggingStream(), + level = logging.DEBUG, + format = '%(name)-12s %(levelname)-8s %(message)s'); + # + # Introspection and configuration. + # + + def isLocal(self): + """Is this a local reporter?""" + return False; + + def incVerbosity(self): + """Increases the verbosity level.""" + self.iVerbose += 1; + + def incDebug(self): + """Increases the debug level.""" + self.iDebug += 1; + + def getVerbosity(self): + """Returns the current verbosity level.""" + return self.iVerbose; + + def getDebug(self): + """Returns the current debug level.""" + return self.iDebug; + + def appendToProcessName(self, sAppend): + """ + Appends sAppend to the base process name. + Returns the new process name. + """ + self.sName = os.path.splitext(os.path.basename(sys.argv[0]))[0] + sAppend; + return self.sName; + + + # + # Generic logging. + # + + def log(self, iLevel, sText, sCaller, sTsPrf): + """ + Writes the specfied text to the log if iLevel is less or requal + to iVerbose. + """ + _ = iLevel; _ = sText; _ = sCaller; _ = sTsPrf; + return 0; + + # + # XML output from the reporter. + # + + def _xmlEscAttr(self, sValue): + """Escapes an XML attribute value.""" + sValue = sValue.replace('&', '&'); + sValue = sValue.replace('<', '<'); + sValue = sValue.replace('>', '>'); + #sValue = sValue.replace('\'', '''); + sValue = sValue.replace('"', '"'); + sValue = sValue.replace('\n', '
'); + sValue = sValue.replace('\r', '
'); + return sValue; + + def _xmlWrite(self, asText, fIndent = True): + """XML output function for the reporter.""" + _ = asText; _ = fIndent; + return None; + + def xmlFlush(self, fRetry = False, fForce = False): + """Flushes XML output if buffered.""" + _ = fRetry; _ = fForce; + return True; + + # + # XML output from child. + # + + def subXmlStart(self, oFileWrapper): + """Called by the file wrapper when the first bytes are written to the test pipe.""" + _ = oFileWrapper; + return None; + + def subXmlWrite(self, oFileWrapper, sRawXml, sCaller): + """Called by the file wrapper write method for test pipes.""" + return self.log(0, 'raw xml%s: %s' % (oFileWrapper.sPrefix, sRawXml), sCaller, utils.getTimePrefix()); + + def subXmlEnd(self, oFileWrapper): + """Called by the file wrapper __del__ method for test pipes.""" + _ = oFileWrapper; + return None; + + # + # File output. + # + + def addLogFile(self, oSrcFile, sSrcFilename, sAltName, sDescription, sKind, sCaller, sTsPrf): + """ + Adds the file to the report. + Returns True on success, False on failure. + """ + _ = oSrcFile; _ = sSrcFilename; _ = sAltName; _ = sDescription; _ = sKind; _ = sCaller; _ = sTsPrf; + return True; + + def addLogString(self, sLog, sLogName, sDescription, sKind, sCaller, sTsPrf): + """ + Adds the file to the report. + Returns True on success, False on failure. + """ + _ = sLog; _ = sLogName; _ = sDescription; _ = sKind; _ = sCaller; _ = sTsPrf; + return True; + + # + # Test reporting + # + + def _testGetFullName(self): + """ + Mangles the test names in atTest into a single name to make it easier + to spot where we are. + """ + sName = ''; + for t in self.atTests: + if sName != '': + sName += ', '; + sName += t[0]; + return sName; + + def testIncErrors(self): + """Increates the error count.""" + self.cErrors += 1; + return self.cErrors; + + def testSetTimedOut(self): + """Sets time out indicator for the current test and increases the error counter.""" + self.fTimedOut = True; + self.cErrors += 1; + return None; + + def testStart(self, sName, sCaller): + """ Starts a new test, may be nested. """ + (sTsPrf, sTsIso) = utils.getTimePrefixAndIsoTimestamp(); + self._xmlWrite([ '<Test timestamp="%s" name="%s">' % (sTsIso, self._xmlEscAttr(sName),), ]); + self.atTests.append((sName, self.cErrors, self.fTimedOut)); + self.fTimedOut = False; + return self.log(1, ' %-50s: TESTING' % (self._testGetFullName()), sCaller, sTsPrf); + + def testValue(self, sName, sValue, sUnit, sCaller): + """ Reports a benchmark value or something simiarlly useful. """ + (sTsPrf, sTsIso) = utils.getTimePrefixAndIsoTimestamp(); + self._xmlWrite([ '<Value timestamp="%s" name="%s" unit="%s" value="%s"/>' + % (sTsIso, self._xmlEscAttr(sName), self._xmlEscAttr(sUnit), self._xmlEscAttr(sValue)), ]); + return self.log(0, '** %-48s: %12s %s' % (sName, sValue, sUnit), sCaller, sTsPrf); + + def testFailure(self, sDetails, sCaller): + """ Reports a failure. """ + (sTsPrf, sTsIso) = utils.getTimePrefixAndIsoTimestamp(); + self.cErrors = self.cErrors + 1; + self._xmlWrite([ '<FailureDetails timestamp="%s" text="%s"/>' % (sTsIso, self._xmlEscAttr(sDetails),), ]); + return self.log(0, sDetails, sCaller, sTsPrf); + + def testDone(self, fSkipped, sCaller): + """ + Marks the current test as DONE, pops it and maks the next test on the + stack current. + Returns (name, errors). + """ + (sTsPrf, sTsIso) = utils.getTimePrefixAndIsoTimestamp(); + sFullName = self._testGetFullName(); + + # safe pop + if not self.atTests: + self.log(0, 'testDone on empty test stack!', sCaller, sTsPrf); + return ('internal error', 0); + fTimedOut = self.fTimedOut; + sName, cErrorsStart, self.fTimedOut = self.atTests.pop(); + + # log + xml. + cErrors = self.cErrors - cErrorsStart; + if cErrors == 0: + if fSkipped is not True: + self._xmlWrite([ ' <Passed timestamp="%s"/>' % (sTsIso,), '</Test>' ],); + self.log(1, '** %-50s: PASSED' % (sFullName,), sCaller, sTsPrf); + else: + self._xmlWrite([ ' <Skipped timestamp="%s"/>' % (sTsIso,), '</Test>' ]); + self.log(1, '** %-50s: SKIPPED' % (sFullName,), sCaller, sTsPrf); + elif fTimedOut: + self._xmlWrite([ ' <TimedOut timestamp="%s" errors="%d"/>' % (sTsIso, cErrors), '</Test>' ]); + self.log(0, '** %-50s: TIMED-OUT - %d errors' % (sFullName, cErrors), sCaller, sTsPrf); + else: + self._xmlWrite([ ' <Failed timestamp="%s" errors="%d"/>' % (sTsIso, cErrors), '</Test>' ]); + self.log(0, '** %-50s: FAILED - %d errors' % (sFullName, cErrors), sCaller, sTsPrf); + + # Flush buffers when reaching the last test. + if not self.atTests: + self.xmlFlush(fRetry = True); + + return (sName, cErrors); + + def testErrorCount(self): + """ + Returns the number of errors accumulated by the current test. + """ + cTests = len(self.atTests); + if cTests <= 0: + return self.cErrors; + return self.cErrors - self.atTests[cTests - 1][1]; + + def testCleanup(self, sCaller): + """ + Closes all open test as failed. + Returns True if no open tests, False if there were open tests. + """ + if not self.atTests: + return True; + for _ in range(len(self.atTests)): + self.testFailure('Test not closed by test drver', sCaller) + self.testDone(False, sCaller); + return False; + + # + # Misc. + # + + def doPollWork(self, sDebug = None): + """ + Check if any pending stuff expired and needs doing. + """ + _ = sDebug; + return None; + + + + +class LocalReporter(ReporterBase): + """ + Local reporter instance. + """ + + def __init__(self): + ReporterBase.__init__(self); + self.oLogFile = None; + self.oXmlFile = None; + self.fXmlOk = True; + self.iSubXml = 0; + self.iOtherFile = 0; + self.fnGetIsoTimestamp = utils.getIsoTimestamp; # Hack to get a timestamp in __del__. + self.oStdErr = sys.stderr; # Hack for __del__ output. + + # + # Figure the main log directory. + # + try: + self.sDefLogDir = os.path.abspath(os.path.expanduser(os.path.join('~', 'VBoxTestLogs'))); + except: + self.sDefLogDir = os.path.abspath("VBoxTestLogs"); + try: + sLogDir = os.path.abspath(os.environ.get('TESTBOX_REPORTER_LOG_DIR', self.sDefLogDir)); + if not os.path.isdir(sLogDir): + os.makedirs(sLogDir, 0o750); + except: + sLogDir = self.sDefLogDir; + if not os.path.isdir(sLogDir): + os.makedirs(sLogDir, 0o750); + + # + # Make a subdirectory for this test run. + # + sTs = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H-%M-%S.log'); + self.sLogDir = sLogDir = os.path.join(sLogDir, '%s-%s' % (sTs, self.sName)); + try: + os.makedirs(self.sLogDir, 0o750); + except: + self.sLogDir = '%s-%s' % (self.sLogDir, os.getpid()); + os.makedirs(self.sLogDir, 0o750); + + # + # Open the log file and write a header. + # + sLogName = os.path.join(self.sLogDir, 'testsuite.log'); + sTsIso = utils.getIsoTimestamp(); + if sys.version_info[0] >= 3: # Add 'b' to prevent write taking issue with encode('utf-8') not returning a string. + self.oLogFile = utils.openNoInherit(sLogName, "wb"); + else: + self.oLogFile = utils.openNoInherit(sLogName, "w"); + self.oLogFile.write(('Created log file at %s.\nRunning: %s' % (sTsIso, sys.argv)).encode('utf-8')); + + # + # Open the xml log file and write the mandatory introduction. + # + # Note! This is done here and not in the base class because the remote + # logger doesn't really need this. It doesn't need the outer + # test wrapper either. + # + sXmlName = os.path.join(self.sLogDir, 'testsuite.xml'); + if sys.version_info[0] >= 3: # Add 'b' to prevent write taking issue with encode('utf-8') not returning a string. + self.oXmlFile = utils.openNoInherit(sXmlName, "wb"); + else: + self.oXmlFile = utils.openNoInherit(sXmlName, "w"); + self._xmlWrite([ '<?xml version="1.0" encoding="UTF-8" ?>', + '<Test timestamp="%s" name="%s">' % (sTsIso, self._xmlEscAttr(self.sName),), ], + fIndent = False); + + def __del__(self): + """Ends and completes the log files.""" + try: sTsIso = self.fnGetIsoTimestamp(); + except Exception as oXcpt: + sTsIso = str(oXcpt); + + if self.oLogFile is not None: + try: + self.oLogFile.write(('\nThe End %s\n' % (sTsIso,)).encode('utf-8')); + self.oLogFile.close(); + except: pass; + self.oLogFile = None; + + if self.oXmlFile is not None: + self._closeXml(sTsIso); + self.oXmlFile = None; + + def _closeXml(self, sTsIso): + """Closes the XML file.""" + if self.oXmlFile is not None: + # pop the test stack + while self.atTests: + sName, cErrorsStart, self.fTimedOut = self.atTests.pop(); + self._xmlWrite([ '<End timestamp="%s" errors="%d"/>' % (sTsIso, self.cErrors - cErrorsStart,), + '</%s>' % (sName,), ]); + + # The outer one is not on the stack. + self._xmlWrite([ ' <End timestamp="%s"/>' % (sTsIso,), + '</Test>', ], fIndent = False); + try: + self.oXmlFile.close(); + self.oXmlFile = None; + except: + pass; + + def _xmlWrite(self, asText, fIndent = True): + """Writes to the XML file.""" + for sText in asText: + if fIndent: + sIndent = ''.ljust((len(self.atTests) + 1) * 2); + sText = sIndent + sText; + sText += '\n'; + + try: + self.oXmlFile.write(sText.encode('utf-8')); + except: + if self.fXmlOk: + traceback.print_exc(); + self.fXmlOk = False; + return False; + return True; + + # + # Overridden methods. + # + + def isLocal(self): + """Is this a local reporter?""" + return True; + + def log(self, iLevel, sText, sCaller, sTsPrf): + if iLevel <= self.iVerbose: + # format it. + if self.iDebug <= 0: + sLogText = '%s %s' % (sTsPrf, sText); + elif self.iDebug <= 1: + sLogText = '%s %30s: %s' % (sTsPrf, sCaller, sText); + else: + sLogText = '%s e=%u %30s: %s' % (sTsPrf, self.cErrors, sCaller, sText); + + # output it. + if sys.version_info[0] >= 3: + sAscii = sLogText; + else: + sAscii = sLogText.encode('ascii', 'replace'); + if self.iDebug == 0: + print('%s: %s' % (self.sName, sAscii), file = self.oStdErr); + else: + print('%s' % (sAscii), file = self.oStdErr); + sLogText += '\n'; + try: + self.oLogFile.write(sLogText.encode('utf-8')); + except: + pass; + return 0; + + def addLogFile(self, oSrcFile, sSrcFilename, sAltName, sDescription, sKind, sCaller, sTsPrf): + # Figure the destination filename. + iOtherFile = self.iOtherFile; + self.iOtherFile += 1; + sDstFilename = os.path.join(self.sLogDir, 'other-%d-%s.log' \ + % (iOtherFile, os.path.splitext(os.path.basename(sSrcFilename))[0])); + self.log(0, '** Other log file: %s - %s (%s)' % (sDstFilename, sDescription, sSrcFilename), sCaller, sTsPrf); + + # Open the destination file and copy over the data. + fRc = True; + try: + oDstFile = utils.openNoInherit(sDstFilename, 'wb'); + except Exception as oXcpt: + self.log(0, 'error opening %s: %s' % (sDstFilename, oXcpt), sCaller, sTsPrf); + else: + while True: + try: + abBuf = oSrcFile.read(65536); + except Exception as oXcpt: + fRc = False; + self.log(0, 'error reading %s: %s' % (sSrcFilename, oXcpt), sCaller, sTsPrf); + else: + try: + oDstFile.write(abBuf); + except Exception as oXcpt: + fRc = False; + self.log(0, 'error writing %s: %s' % (sDstFilename, oXcpt), sCaller, sTsPrf); + else: + if abBuf: + continue; + break; + oDstFile.close(); + + # Leave a mark in the XML log. + self._xmlWrite(['<LogFile timestamp="%s" filename="%s" source="%s" kind="%s" ok="%s">%s</LogFile>\n' + % (utils.getIsoTimestamp(), self._xmlEscAttr(os.path.basename(sDstFilename)), self._xmlEscAttr(sSrcFilename), \ + self._xmlEscAttr(sKind), fRc, self._xmlEscAttr(sDescription))] ); + _ = sAltName; + return fRc; + + def addLogString(self, sLog, sLogName, sDescription, sKind, sCaller, sTsPrf): + # Figure the destination filename. + iOtherFile = self.iOtherFile; + self.iOtherFile += 1; + sDstFilename = os.path.join(self.sLogDir, 'other-%d-%s.log' \ + % (iOtherFile, os.path.splitext(os.path.basename(sLogName))[0])); + self.log(0, '** Other log file: %s - %s (%s)' % (sDstFilename, sDescription, sLogName), sCaller, sTsPrf); + + # Open the destination file and copy over the data. + fRc = True; + try: + oDstFile = utils.openNoInherit(sDstFilename, 'w'); + except Exception as oXcpt: + self.log(0, 'error opening %s: %s' % (sDstFilename, oXcpt), sCaller, sTsPrf); + else: + try: + oDstFile.write(sLog); + except Exception as oXcpt: + fRc = False; + self.log(0, 'error writing %s: %s' % (sDstFilename, oXcpt), sCaller, sTsPrf); + + oDstFile.close(); + + # Leave a mark in the XML log. + self._xmlWrite(['<LogFile timestamp="%s" filename="%s" source="%s" kind="%s" ok="%s">%s</LogFile>\n' + % (utils.getIsoTimestamp(), self._xmlEscAttr(os.path.basename(sDstFilename)), self._xmlEscAttr(sLogName), \ + self._xmlEscAttr(sKind), fRc, self._xmlEscAttr(sDescription))] ); + return fRc; + + def subXmlStart(self, oFileWrapper): + # Open a new file and just include it from the main XML. + iSubXml = self.iSubXml; + self.iSubXml += 1; + sSubXmlName = os.path.join(self.sLogDir, 'sub-%d.xml' % (iSubXml,)); + try: + oFileWrapper.oSubXmlFile = utils.openNoInherit(sSubXmlName, "w"); + except: + errorXcpt('open(%s)' % oFileWrapper.oSubXmlName); + oFileWrapper.oSubXmlFile = None; + else: + self._xmlWrite(['<Include timestamp="%s" filename="%s"/>\n' + % (utils.getIsoTimestamp(), self._xmlEscAttr(os.path.basename(sSubXmlName)))]); + return None; + + def subXmlWrite(self, oFileWrapper, sRawXml, sCaller): + if oFileWrapper.oSubXmlFile is not None: + try: + oFileWrapper.oSubXmlFile.write(sRawXml); + except: + pass; + if sCaller is None: pass; # pychecker - NOREF + return None; + + def subXmlEnd(self, oFileWrapper): + if oFileWrapper.oSubXmlFile is not None: + try: + oFileWrapper.oSubXmlFile.close(); + oFileWrapper.oSubXmlFile = None; + except: + pass; + return None; + + + +class RemoteReporter(ReporterBase): + """ + Reporter that talks to the test manager server. + """ + + + ## The XML sync min time (seconds). + kcSecXmlFlushMin = 30; + ## The XML sync max time (seconds). + kcSecXmlFlushMax = 120; + ## The XML sync idle time before flushing (seconds). + kcSecXmlFlushIdle = 5; + ## The XML sync line count threshold. + kcLinesXmlFlush = 512; + + ## The retry timeout. + kcSecTestManagerRetryTimeout = 120; + ## The request timeout. + kcSecTestManagerRequestTimeout = 30; + + + def __init__(self): + ReporterBase.__init__(self); + self.sTestManagerUrl = os.environ.get('TESTBOX_MANAGER_URL'); + self.sTestBoxUuid = os.environ.get('TESTBOX_UUID'); + self.idTestBox = int(os.environ.get('TESTBOX_ID')); + self.idTestSet = int(os.environ.get('TESTBOX_TEST_SET_ID')); + self._asXml = []; + self._secTsXmlFlush = utils.timestampSecond(); + self._secTsXmlLast = self._secTsXmlFlush; + self._fXmlFlushing = False; + self.oOutput = sys.stdout; # Hack for __del__ output. + self.fFlushEachLine = True; + self.fDebugXml = 'TESTDRIVER_REPORTER_DEBUG_XML' in os.environ; + + # Prepare the TM connecting. + from common import constants; + if sys.version_info[0] >= 3: + import urllib; + self._fnUrlEncode = urllib.parse.urlencode; # pylint: disable=no-member + self._fnUrlParseQs = urllib.parse.parse_qs; # pylint: disable=no-member + self._oParsedTmUrl = urllib.parse.urlparse(self.sTestManagerUrl); # pylint: disable=no-member + import http.client as httplib; # pylint: disable=no-name-in-module,import-error + else: + import urllib; + self._fnUrlEncode = urllib.urlencode; # pylint: disable=no-member + import urlparse; # pylint: disable=import-error + self._fnUrlParseQs = urlparse.parse_qs; # pylint: disable=no-member + self._oParsedTmUrl = urlparse.urlparse(self.sTestManagerUrl); # pylint: disable=no-member + import httplib; # pylint: disable=no-name-in-module,import-error + + if sys.version_info[0] >= 3 \ + or (sys.version_info[0] == 2 and sys.version_info[1] >= 6): + if self._oParsedTmUrl.scheme == 'https': # pylint: disable=no-member + self._fnTmConnect = lambda: httplib.HTTPSConnection(self._oParsedTmUrl.hostname, + timeout = self.kcSecTestManagerRequestTimeout); + else: + self._fnTmConnect = lambda: httplib.HTTPConnection( self._oParsedTmUrl.hostname, + timeout = self.kcSecTestManagerRequestTimeout); + else: + if self._oParsedTmUrl.scheme == 'https': # pylint: disable=no-member + self._fnTmConnect = lambda: httplib.HTTPSConnection(self._oParsedTmUrl.hostname); + else: + self._fnTmConnect = lambda: httplib.HTTPConnection( self._oParsedTmUrl.hostname); + self._dHttpHeader = \ + { + 'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8', + 'User-Agent': 'TestDriverReporter/%s.0 (%s, %s)' % (__version__, utils.getHostOs(), utils.getHostArch(),), + 'Accept': 'text/plain,application/x-www-form-urlencoded', + 'Accept-Encoding': 'identity', + 'Cache-Control': 'max-age=0', + #'Connection': 'keep-alive', + }; + + dParams = { + constants.tbreq.ALL_PARAM_TESTBOX_UUID: self.sTestBoxUuid, + constants.tbreq.ALL_PARAM_TESTBOX_ID: self.idTestBox, + constants.tbreq.RESULT_PARAM_TEST_SET_ID: self.idTestSet, + }; + self._sTmServerPath = '/%s/testboxdisp.py?%s' \ + % ( self._oParsedTmUrl.path.strip('/'), # pylint: disable=no-member + self._fnUrlEncode(dParams), ); + + def __del__(self): + """Flush pending log messages?""" + if self._asXml: + self._xmlDoFlush(self._asXml, fRetry = True, fDtor = True); + + def _writeOutput(self, sText): + """ Does the actual writing and flushing. """ + if sys.version_info[0] >= 3: + print(sText, file = self.oOutput); + else: + print(sText.encode('ascii', 'replace'), file = self.oOutput); + if self.fFlushEachLine: self.oOutput.flush(); + return None; + + # + # Talking to TM. + # + + def _processTmStatusResponse(self, oConn, sOperation, fClose = True): + """ + Processes HTTP reponse from the test manager. + Returns True, False or None. None should be retried, the others not. + May raise exception on HTTP issue (retry ok). + """ + if sys.version_info[0] >= 3: import http.client as httplib; # pylint: disable=no-name-in-module,import-error + else: import httplib; # pylint: disable=import-error + from common import constants; + + # Read the response and (optionally) close the connection. + oResponse = oConn.getresponse(); + try: + sRspBody = oResponse.read(); + except httplib.IncompleteRead as oXcpt: + self._writeOutput('%s: %s: Warning: httplib.IncompleteRead: %s [expected %s, got %s]' + % (utils.getTimePrefix(), sOperation, oXcpt, oXcpt.expected, len(oXcpt.partial),)); + sRspBody = oXcpt.partial; + if fClose is True: + try: oConn.close(); + except: pass; + + # Make sure it's a string which encoding we grok. + if hasattr(sRspBody, 'decode'): + sRspBody = sRspBody.decode('utf-8', 'ignore'); + + # Check the content type. + sContentType = oResponse.getheader('Content-Type'); + if sContentType is not None and sContentType == 'application/x-www-form-urlencoded; charset=utf-8': + + # Parse the body and check the RESULT parameter. + dResponse = self._fnUrlParseQs(sRspBody, strict_parsing = True); + sResult = dResponse.get(constants.tbresp.ALL_PARAM_RESULT, None); + if isinstance(sResult, list): + sResult = sResult[0] if len(sResult) == 1 else '%d results' % (len(sResult),); + + if sResult is not None: + if sResult == constants.tbresp.STATUS_ACK: + return True; + if sResult == constants.tbresp.STATUS_NACK: + self._writeOutput('%s: %s: Failed (%s). (dResponse=%s)' + % (utils.getTimePrefix(), sOperation, sResult, dResponse,)); + return False; + + self._writeOutput('%s: %s: Failed - dResponse=%s' % (utils.getTimePrefix(), sOperation, dResponse,)); + else: + self._writeOutput('%s: %s: Unexpected Content-Type: %s' % (utils.getTimePrefix(), sOperation, sContentType,)); + self._writeOutput('%s: %s: Body: %s' % (utils.getTimePrefix(), sOperation, sRspBody,)); + return None; + + def _doUploadFile(self, oSrcFile, sSrcFilename, sDescription, sKind, sMime): + """ Uploads the given file to the test manager. """ + + # Prepare header and url. + dHeader = dict(self._dHttpHeader); + dHeader['Content-Type'] = 'application/octet-stream'; + self._writeOutput('%s: _doUploadFile: sHeader=%s' % (utils.getTimePrefix(), dHeader,)); + oSrcFile.seek(0, 2); + cbFileSize = oSrcFile.tell(); + self._writeOutput('%s: _doUploadFile: size=%d' % (utils.getTimePrefix(), cbFileSize,)); + oSrcFile.seek(0); + + if cbFileSize <= 0: # The Test Manager will bitch if the file size is 0, so skip uploading. + self._writeOutput('%s: _doUploadFile: Empty file, skipping upload' % utils.getTimePrefix()); + return False; + + from common import constants; + sUrl = self._sTmServerPath + '&' \ + + self._fnUrlEncode({ constants.tbreq.UPLOAD_PARAM_NAME: os.path.basename(sSrcFilename), + constants.tbreq.UPLOAD_PARAM_DESC: sDescription, + constants.tbreq.UPLOAD_PARAM_KIND: sKind, + constants.tbreq.UPLOAD_PARAM_MIME: sMime, + constants.tbreq.ALL_PARAM_ACTION: constants.tbreq.UPLOAD, + }); + + # Retry loop. + secStart = utils.timestampSecond(); + while True: + try: + oConn = self._fnTmConnect(); + oConn.request('POST', sUrl, oSrcFile.read(), dHeader); + fRc = self._processTmStatusResponse(oConn, '_doUploadFile', fClose = True); + oConn.close(); + if fRc is not None: + return fRc; + except: + logXcpt('warning: exception during UPLOAD request'); + + if utils.timestampSecond() - secStart >= self.kcSecTestManagerRetryTimeout: + self._writeOutput('%s: _doUploadFile: Timed out.' % (utils.getTimePrefix(),)); + break; + try: oSrcFile.seek(0); + except: + logXcpt(); + break; + self._writeOutput('%s: _doUploadFile: Retrying...' % (utils.getTimePrefix(), )); + time.sleep(2); + + return False; + + def _doUploadString(self, sSrc, sSrcName, sDescription, sKind, sMime): + """ Uploads the given string as a separate file to the test manager. """ + + # Prepare header and url. + dHeader = dict(self._dHttpHeader); + dHeader['Content-Type'] = 'application/octet-stream'; + self._writeOutput('%s: _doUploadString: sHeader=%s' % (utils.getTimePrefix(), dHeader,)); + self._writeOutput('%s: _doUploadString: size=%d' % (utils.getTimePrefix(), sys.getsizeof(sSrc),)); + + from common import constants; + sUrl = self._sTmServerPath + '&' \ + + self._fnUrlEncode({ constants.tbreq.UPLOAD_PARAM_NAME: os.path.basename(sSrcName), + constants.tbreq.UPLOAD_PARAM_DESC: sDescription, + constants.tbreq.UPLOAD_PARAM_KIND: sKind, + constants.tbreq.UPLOAD_PARAM_MIME: sMime, + constants.tbreq.ALL_PARAM_ACTION: constants.tbreq.UPLOAD, + }); + + # Retry loop. + secStart = utils.timestampSecond(); + while True: + try: + oConn = self._fnTmConnect(); + oConn.request('POST', sUrl, sSrc, dHeader); + fRc = self._processTmStatusResponse(oConn, '_doUploadString', fClose = True); + oConn.close(); + if fRc is not None: + return fRc; + except: + logXcpt('warning: exception during UPLOAD request'); + + if utils.timestampSecond() - secStart >= self.kcSecTestManagerRetryTimeout: + self._writeOutput('%s: _doUploadString: Timed out.' % (utils.getTimePrefix(),)); + break; + self._writeOutput('%s: _doUploadString: Retrying...' % (utils.getTimePrefix(), )); + time.sleep(2); + + return False; + + def _xmlDoFlush(self, asXml, fRetry = False, fDtor = False): + """ + The code that does the actual talking to the server. + Used by both xmlFlush and __del__. + """ + secStart = utils.timestampSecond(); + while True: + fRc = None; + try: + # Post. + from common import constants; + sPostBody = self._fnUrlEncode({constants.tbreq.XML_RESULT_PARAM_BODY: '\n'.join(asXml),}); + oConn = self._fnTmConnect(); + oConn.request('POST', + self._sTmServerPath + ('&%s=%s' % (constants.tbreq.ALL_PARAM_ACTION, constants.tbreq.XML_RESULTS)), + sPostBody, + self._dHttpHeader); + + fRc = self._processTmStatusResponse(oConn, '_xmlDoFlush', fClose = True); + if fRc is True: + if self.fDebugXml: + self._writeOutput('_xmlDoFlush:\n%s' % ('\n'.join(asXml),)); + return (None, False); + if fRc is False: + self._writeOutput('_xmlDoFlush: Failed - we should abort the test, really.'); + return (None, True); + except Exception as oXcpt: + if not fDtor: + logXcpt('warning: exception during XML_RESULTS request'); + else: + self._writeOutput('warning: exception during XML_RESULTS request: %s' % (oXcpt,)); + + if fRetry is not True \ + or utils.timestampSecond() - secStart >= self.kcSecTestManagerRetryTimeout: + break; + time.sleep(2); + + return (asXml, False); + + + # + # Overridden methods. + # + + def isLocal(self): + return False; + + def log(self, iLevel, sText, sCaller, sTsPrf): + if iLevel <= self.iVerbose: + if self.iDebug <= 0: + sLogText = '%s %s' % (sTsPrf, sText); + elif self.iDebug <= 1: + sLogText = '%s %30s: %s' % (sTsPrf, sCaller, sText); + else: + sLogText = '%s e=%u %30s: %s' % (sTsPrf, self.cErrors, sCaller, sText); + self._writeOutput(sLogText); + return 0; + + def addLogFile(self, oSrcFile, sSrcFilename, sAltName, sDescription, sKind, sCaller, sTsPrf): + fRc = True; + if sKind in [ 'text', 'log', 'process'] \ + or sKind.startswith('log/') \ + or sKind.startswith('info/') \ + or sKind.startswith('process/'): + self.log(0, '*** Uploading "%s" - KIND: "%s" - DESC: "%s" ***' + % (sSrcFilename, sKind, sDescription), sCaller, sTsPrf); + self.xmlFlush(); + g_oLock.release(); + try: + self._doUploadFile(oSrcFile, sAltName, sDescription, sKind, 'text/plain'); + finally: + g_oLock.acquire(); + elif sKind.startswith('screenshot/'): + self.log(0, '*** Uploading "%s" - KIND: "%s" - DESC: "%s" ***' + % (sSrcFilename, sKind, sDescription), sCaller, sTsPrf); + self.xmlFlush(); + g_oLock.release(); + try: + self._doUploadFile(oSrcFile, sAltName, sDescription, sKind, 'image/png'); + finally: + g_oLock.acquire(); + elif sKind.startswith('screenrecording/'): + self.log(0, '*** Uploading "%s" - KIND: "%s" - DESC: "%s" ***' + % (sSrcFilename, sKind, sDescription), sCaller, sTsPrf); + self.xmlFlush(); + g_oLock.release(); + try: + self._doUploadFile(oSrcFile, sAltName, sDescription, sKind, 'video/webm'); + finally: + g_oLock.acquire(); + elif sKind.startswith('misc/'): + self.log(0, '*** Uploading "%s" - KIND: "%s" - DESC: "%s" ***' + % (sSrcFilename, sKind, sDescription), sCaller, sTsPrf); + self.xmlFlush(); + g_oLock.release(); + try: + self._doUploadFile(oSrcFile, sAltName, sDescription, sKind, 'application/octet-stream'); + finally: + g_oLock.acquire(); + else: + self.log(0, '*** UNKNOWN FILE "%s" - KIND "%s" - DESC "%s" ***' + % (sSrcFilename, sKind, sDescription), sCaller, sTsPrf); + return fRc; + + def addLogString(self, sLog, sLogName, sDescription, sKind, sCaller, sTsPrf): + fRc = True; + if sKind in [ 'text', 'log', 'process'] \ + or sKind.startswith('log/') \ + or sKind.startswith('info/') \ + or sKind.startswith('process/'): + self.log(0, '*** Uploading "%s" - KIND: "%s" - DESC: "%s" ***' + % (sLogName, sKind, sDescription), sCaller, sTsPrf); + self.xmlFlush(); + g_oLock.release(); + try: + self._doUploadString(sLog, sLogName, sDescription, sKind, 'text/plain'); + finally: + g_oLock.acquire(); + else: + self.log(0, '*** UNKNOWN FILE "%s" - KIND "%s" - DESC "%s" ***' + % (sLogName, sKind, sDescription), sCaller, sTsPrf); + return fRc; + + def xmlFlush(self, fRetry = False, fForce = False): + """ + Flushes the XML back log. Called with the lock held, may leave it + while communicating with the server. + """ + if not self._fXmlFlushing: + asXml = self._asXml; + self._asXml = []; + if asXml or fForce is True: + self._fXmlFlushing = True; + + g_oLock.release(); + try: + (asXml, fIncErrors) = self._xmlDoFlush(asXml, fRetry = fRetry); + finally: + g_oLock.acquire(); + + if fIncErrors: + self.testIncErrors(); + + self._fXmlFlushing = False; + if asXml is None: + self._secTsXmlFlush = utils.timestampSecond(); + else: + self._asXml = asXml + self._asXml; + return True; + + self._secTsXmlFlush = utils.timestampSecond(); + return False; + + def _xmlFlushIfNecessary(self, fPolling = False, sDebug = None): + """Flushes the XML back log if necessary.""" + tsNow = utils.timestampSecond(); + cSecs = tsNow - self._secTsXmlFlush; + cSecsLast = tsNow - self._secTsXmlLast; + if fPolling is not True: + self._secTsXmlLast = tsNow; + + # Absolute flush thresholds. + if cSecs >= self.kcSecXmlFlushMax: + return self.xmlFlush(); + if len(self._asXml) >= self.kcLinesXmlFlush: + return self.xmlFlush(); + + # Flush if idle long enough. + if cSecs >= self.kcSecXmlFlushMin \ + and cSecsLast >= self.kcSecXmlFlushIdle: + return self.xmlFlush(); + + _ = sDebug; + return False; + + def _xmlWrite(self, asText, fIndent = True): + """XML output function for the reporter.""" + self._asXml += asText; + self._xmlFlushIfNecessary(); + _ = fIndent; # No pretty printing, thank you. + return None; + + def subXmlStart(self, oFileWrapper): + oFileWrapper.sXmlBuffer = ''; + return None; + + def subXmlWrite(self, oFileWrapper, sRawXml, sCaller): + oFileWrapper.sXmlBuffer += sRawXml; + _ = sCaller; + return None; + + def subXmlEnd(self, oFileWrapper): + sRawXml = oFileWrapper.sXmlBuffer; + ## @todo should validate the document here and maybe auto terminate things. Adding some hints to have the server do + # this instead. + g_oLock.acquire(); + try: + self._asXml += [ '<PushHint testdepth="%d"/>' % (len(self.atTests),), + sRawXml, + '<PopHint testdepth="%d"/>' % (len(self.atTests),),]; + self._xmlFlushIfNecessary(); + finally: + g_oLock.release(); + return None; + + def doPollWork(self, sDebug = None): + if self._asXml: + g_oLock.acquire(); + try: + self._xmlFlushIfNecessary(fPolling = True, sDebug = sDebug); + finally: + g_oLock.release(); + return None; + + +# +# Helpers +# + +g_fnComXcptFormatter = None; + +def setComXcptFormatter(fnCallback): + """ + Install callback for prettier COM exception formatting. + + The callback replaces the work done by format_exception_only() and + takes the same arguments. It returns None if not interested in the + exception. + """ + global g_fnComXcptFormatter; + g_fnComXcptFormatter = fnCallback; + return True; + +def formatExceptionOnly(oType, oXcpt, sCaller, sTsPrf): + """ + Wrapper around traceback.format_exception_only and __g_fnComXcptFormatter. + """ + #asRet = ['oType=%s type(oXcpt)=%s' % (oType, type(oXcpt),)]; + asRet = []; + + # Try the callback first. + fnCallback = g_fnComXcptFormatter; + if fnCallback: + try: + asRetCb = fnCallback(oType, oXcpt); + if asRetCb: + return asRetCb; + #asRet += asRetCb; + except: + g_oReporter.log(0, '** internal-error: Hit exception #2 in __g_fnComXcptFormatter! %s' + % (traceback.format_exc()), sCaller, sTsPrf); + asRet += ['internal error: exception in __g_fnComXcptFormatter']; + + # Now try format_exception_only: + try: + asRet += traceback.format_exception_only(oType, oXcpt); + except: + g_oReporter.log(0, '** internal-error: Hit exception #2 in format_exception_only! %s' + % (traceback.format_exc()), sCaller, sTsPrf); + asRet += ['internal error: Exception in format_exception_only!']; + return asRet; + + +def logXcptWorker(iLevel, fIncErrors, sPrefix="", sText=None, cFrames=1): + """ + Log an exception, optionally with a preceeding message and more than one + call frame. + """ + g_oLock.acquire(); + try: + + if fIncErrors: + g_oReporter.testIncErrors(); + + ## @todo skip all this if iLevel is too high! + + # Try get exception info. + sTsPrf = utils.getTimePrefix(); + try: + oType, oValue, oTraceback = sys.exc_info(); + except: + oType = oValue = oTraceback = None; + if oType is not None: + + # Try format the info + try: + rc = 0; + sCaller = utils.getCallerName(oTraceback.tb_frame); + if sText is not None: + rc = g_oReporter.log(iLevel, "%s%s" % (sPrefix, sText), sCaller, sTsPrf); + asInfo = None; + try: + asInfo = formatExceptionOnly(oType, oValue, sCaller, sTsPrf); + atEntries = traceback.extract_tb(oTraceback); + atEntries.reverse(); + if cFrames is not None and cFrames <= 1: + if atEntries: + asInfo = asInfo + traceback.format_list(atEntries[:1]); + else: + asInfo.append('Traceback (stack order):') + if cFrames is not None and cFrames < len(atEntries): + asInfo = asInfo + traceback.format_list(atEntries[:cFrames]); + else: + asInfo = asInfo + traceback.format_list(atEntries); + asInfo.append('Stack:') + asInfo = asInfo + traceback.format_stack(oTraceback.tb_frame.f_back, cFrames); + except: + g_oReporter.log(0, '** internal-error: Hit exception #2! %s' % (traceback.format_exc()), sCaller, sTsPrf); + + if asInfo: + # Do the logging. + for sItem in asInfo: + asLines = sItem.splitlines(); + for sLine in asLines: + rc = g_oReporter.log(iLevel, '%s%s' % (sPrefix, sLine), sCaller, sTsPrf); + + else: + g_oReporter.log(iLevel, 'No exception info...', sCaller, sTsPrf); + rc = -3; + except: + g_oReporter.log(0, '** internal-error: Hit exception! %s' % (traceback.format_exc()), None, sTsPrf); + rc = -2; + else: + g_oReporter.log(0, '** internal-error: No exception! %s' + % (utils.getCallerName(iFrame=3)), utils.getCallerName(iFrame=3), sTsPrf); + rc = -1; + + finally: + g_oLock.release(); + return rc; + + +# +# The public Classes +# +class FileWrapper(object): + """ File like class for TXS EXEC and similar. """ + def __init__(self, sPrefix): + self.sPrefix = sPrefix; + + def __del__(self): + self.close(); + + def close(self): + """ file.close """ + # Nothing to be done. + return; + + def read(self, cb): + """file.read""" + _ = cb; + return ""; + + def write(self, sText): + """file.write""" + if not utils.isString(sText): + if isinstance(sText, array.array): + try: + if sys.version_info < (3, 9, 0): + # Removed since Python 3.9. + sText = sText.tostring(); # pylint: disable=no-member + else: + sText = sText.tobytes(); + except: + pass; + if hasattr(sText, 'decode'): + try: + sText = sText.decode('utf-8', 'ignore'); + except: + pass; + g_oLock.acquire(); + try: + sTsPrf = utils.getTimePrefix(); + sCaller = utils.getCallerName(); + asLines = sText.splitlines(); + for sLine in asLines: + g_oReporter.log(0, '%s: %s' % (self.sPrefix, sLine), sCaller, sTsPrf); + except: + traceback.print_exc(); + finally: + g_oLock.release(); + return None; + +class FileWrapperTestPipe(object): + """ + File like class for the test pipe (TXS EXEC and similar). + + This is also used to submit XML test result files. + """ + def __init__(self): + self.sPrefix = ''; + self.fStarted = False; + self.fClosed = False; + self.sTagBuffer = None; + self.cTestDepth = 0; + self.acTestErrors = []; + + def __del__(self): + self.close(); + + def close(self): + """ file.close """ + if self.fStarted is True and self.fClosed is False: + self.fClosed = True; + + # Close open <Test> elements: + if self.cTestDepth > 0: + sNow = utils.getIsoTimestamp() + cErrors = 0; + while self.cTestDepth > 0: + self.cTestDepth -= 1; + if self.acTestErrors: + cErrors += self.acTestErrors.pop(); + cErrors += 1; + g_oReporter.subXmlWrite(self, + '\n%s <Failed timestamp="%s" errors="%s"/>\n%s</Test>\n' + % (' ' * self.cTestDepth, sNow, cErrors, ' ' * self.cTestDepth), + utils.getCallerName()); + + # Tell the reporter that the XML input is done. + try: g_oReporter.subXmlEnd(self); + except: + try: traceback.print_exc(); + except: pass; + return True; + + def read(self, cb = None): + """file.read""" + _ = cb; + return ""; + + def write(self, sText): + """file.write""" + # lazy start. + if self.fStarted is not True: + try: + g_oReporter.subXmlStart(self); + except: + traceback.print_exc(); + self.fStarted = True; + + # Turn non-string stuff into strings. + if not utils.isString(sText): + if isinstance(sText, array.array): + try: + if sys.version_info < (3, 9, 0): + # Removed since Python 3.9. + sText = sText.tostring(); # pylint: disable=no-member + else: + sText = sText.tobytes(); + except: + pass; + if hasattr(sText, 'decode'): + try: sText = sText.decode('utf-8', 'ignore'); + except: pass; + + try: + # + # Write the XML to the reporter. + # + g_oReporter.subXmlWrite(self, sText, utils.getCallerName()); + + # + # Parse the supplied text and look for <Failed.../> tags to keep track of the + # error counter. This is only a very lazy aproach. + # + idxText = 0; + while sText: + if self.sTagBuffer is None: + # Look for the start of a tag. + idxStart = sText.find('<', idxText); + if idxStart != -1: + # If the end was found inside the current buffer, parse the line, + # otherwise we have to save it for later. + idxEnd = sText.find('>', idxStart); + if idxEnd != -1: + self._processXmlElement(sText[idxStart:idxEnd+1]); + idxText = idxEnd; + else: + self.sTagBuffer = sText[idxStart:]; + break; + else: + break; + else: + # Search for the end of the tag and parse the whole tag. + assert(idxText == 0); + idxEnd = sText.find('>'); + if idxEnd != -1: + self._processXmlElement(self.sTagBuffer + sText[:idxEnd+1]); + self.sTagBuffer = None; + idxText = idxEnd; + else: + self.sTagBuffer = self.sTagBuffer + sText[idxText:]; + break; + except: + traceback.print_exc(); + return None; + + def _processXmlElement(self, sElement): + """ + Processes a complete XML tag. + + We handle the 'Failed' tag to keep track of the error counter. + We also track 'Test' tags to make sure we close with all of them properly closed. + """ + # Make sure we don't parse any space between < and the element name. + sElement = sElement.strip(); + + # Find the end of the name + idxEndName = sElement.find(' '); + if idxEndName == -1: + idxEndName = sElement.find('>'); + if idxEndName >= 0: + if sElement[idxEndName - 1] == '/': + idxEndName -= 1; + else: + idxEndName = len(sElement); + sElementName = sElement[1:idxEndName]; + + # <Failed>: + if sElementName == 'Failed': + g_oLock.acquire(); + try: + g_oReporter.testIncErrors(); + finally: + g_oLock.release(); + if self.acTestErrors: + self.acTestErrors[-1] += 1; # get errors attrib + # <Test> + elif sElementName == 'Test': + self.cTestDepth += 1; + self.acTestErrors.append(0); + # </Test> + elif sElementName == '/Test': + self.cTestDepth -= 1; + if self.acTestErrors: + cErrors = self.acTestErrors.pop(); + if self.acTestErrors: + self.acTestErrors[-1] += cErrors; + + +# +# The public APIs. +# + +def log(sText, sCaller = None): + """Writes the specfied text to the log.""" + g_oLock.acquire(); + try: + rc = g_oReporter.log(1, sText, sCaller if sCaller else utils.getCallerName(), utils.getTimePrefix()); + except: + rc = -1; + finally: + g_oLock.release(); + return rc; + +def logXcpt(sText=None, cFrames=1): + """ + Log an exception, optionally with a preceeding message and more than one + call frame. + """ + return logXcptWorker(1, False, "", sText, cFrames); + +def log2(sText, sCaller = None): + """Log level 2: Writes the specfied text to the log.""" + g_oLock.acquire(); + try: + rc = g_oReporter.log(2, sText, sCaller if sCaller else utils.getCallerName(), utils.getTimePrefix()); + except: + rc = -1; + finally: + g_oLock.release(); + return rc; + +def log2Xcpt(sText=None, cFrames=1): + """ + Log level 2: Log an exception, optionally with a preceeding message and + more than one call frame. + """ + return logXcptWorker(2, False, "", sText, cFrames); + +def log3(sText, sCaller = None): + """Log level 3: Writes the specfied text to the log.""" + g_oLock.acquire(); + try: + rc = g_oReporter.log(3, sText, sCaller if sCaller else utils.getCallerName(), utils.getTimePrefix()); + except: + rc = -1; + finally: + g_oLock.release(); + return rc; + +def log3Xcpt(sText=None, cFrames=1): + """ + Log level 3: Log an exception, optionally with a preceeding message and + more than one call frame. + """ + return logXcptWorker(3, False, "", sText, cFrames); + +def log4(sText, sCaller = None): + """Log level 4: Writes the specfied text to the log.""" + g_oLock.acquire(); + try: + rc = g_oReporter.log(4, sText, sCaller if sCaller else utils.getCallerName(), utils.getTimePrefix()); + except: + rc = -1; + finally: + g_oLock.release(); + return rc; + +def log4Xcpt(sText=None, cFrames=1): + """ + Log level 4: Log an exception, optionally with a preceeding message and + more than one call frame. + """ + return logXcptWorker(4, False, "", sText, cFrames); + +def log5(sText, sCaller = None): + """Log level 2: Writes the specfied text to the log.""" + g_oLock.acquire(); + try: + rc = g_oReporter.log(5, sText, sCaller if sCaller else utils.getCallerName(), utils.getTimePrefix()); + except: + rc = -1; + finally: + g_oLock.release(); + return rc; + +def log5Xcpt(sText=None, cFrames=1): + """ + Log level 5: Log an exception, optionally with a preceeding message and + more than one call frame. + """ + return logXcptWorker(5, False, "", sText, cFrames); + +def log6(sText, sCaller = None): + """Log level 6: Writes the specfied text to the log.""" + g_oLock.acquire(); + try: + rc = g_oReporter.log(6, sText, sCaller if sCaller else utils.getCallerName(), utils.getTimePrefix()); + except: + rc = -1; + finally: + g_oLock.release(); + return rc; + +def log6Xcpt(sText=None, cFrames=1): + """ + Log level 6: Log an exception, optionally with a preceeding message and + more than one call frame. + """ + return logXcptWorker(6, False, "", sText, cFrames); + +def maybeErr(fIsError, sText): + """ Maybe error or maybe normal log entry. """ + if fIsError is True: + return error(sText, sCaller = utils.getCallerName()); + return log(sText, sCaller = utils.getCallerName()); + +def maybeErrXcpt(fIsError, sText=None, cFrames=1): + """ Maybe error or maybe normal log exception entry. """ + if fIsError is True: + return errorXcpt(sText, cFrames); + return logXcpt(sText, cFrames); + +def maybeLog(fIsNotError, sText): + """ Maybe error or maybe normal log entry. """ + if fIsNotError is not True: + return error(sText, sCaller = utils.getCallerName()); + return log(sText, sCaller = utils.getCallerName()); + +def maybeLogXcpt(fIsNotError, sText=None, cFrames=1): + """ Maybe error or maybe normal log exception entry. """ + if fIsNotError is not True: + return errorXcpt(sText, cFrames); + return logXcpt(sText, cFrames); + +def error(sText, sCaller = None): + """ + Writes the specfied error message to the log. + + This will add an error to the current test. + + Always returns False for the convenience of methods returning boolean + success indicators. + """ + g_oLock.acquire(); + try: + g_oReporter.testIncErrors(); + g_oReporter.log(0, '** error: %s' % (sText), sCaller if sCaller else utils.getCallerName(), utils.getTimePrefix()); + except: + pass; + finally: + g_oLock.release(); + return False; + +def errorXcpt(sText=None, cFrames=1): + """ + Log an error caused by an exception. If sText is given, it will preceed + the exception information. cFrames can be used to display more stack. + + This will add an error to the current test. + + Always returns False for the convenience of methods returning boolean + success indicators. + """ + logXcptWorker(0, True, '** error: ', sText, cFrames); + return False; + +def errorTimeout(sText): + """ + Flags the current test as having timed out and writes the specified message to the log. + + This will add an error to the current test. + + Always returns False for the convenience of methods returning boolean + success indicators. + """ + g_oLock.acquire(); + try: + g_oReporter.testSetTimedOut(); + g_oReporter.log(0, '** timeout-error: %s' % (sText), utils.getCallerName(), utils.getTimePrefix()); + except: + pass; + finally: + g_oLock.release(); + return False; + +def fatal(sText): + """ + Writes a fatal error to the log. + + This will add an error to the current test. + + Always returns False for the convenience of methods returning boolean + success indicators. + """ + g_oLock.acquire(); + try: + g_oReporter.testIncErrors(); + g_oReporter.log(0, '** fatal error: %s' % (sText), utils.getCallerName(), utils.getTimePrefix()); + except: + pass + finally: + g_oLock.release(); + return False; + +def fatalXcpt(sText=None, cFrames=1): + """ + Log a fatal error caused by an exception. If sText is given, it will + preceed the exception information. cFrames can be used to display more + stack. + + This will add an error to the current test. + + Always returns False for the convenience of methods returning boolean + success indicators. + """ + logXcptWorker(0, True, "** fatal error: ", sText, cFrames); + return False; + +def addLogFile(sFilename, sKind, sDescription = '', sAltName = None): + """ + Adds the specified log file to the report if the file exists. + + The sDescription is a free form description of the log file. + + The sKind parameter is for adding some machine parsable hint what kind of + log file this really is. + + Returns True on success, False on failure (no ENOENT errors are logged). + """ + sTsPrf = utils.getTimePrefix(); + sCaller = utils.getCallerName(); + fRc = False; + if sAltName is None: + sAltName = sFilename; + + try: + oSrcFile = utils.openNoInherit(sFilename, 'rb'); + except IOError as oXcpt: + if oXcpt.errno != errno.ENOENT: + logXcpt('addLogFile(%s,%s,%s)' % (sFilename, sDescription, sKind)); + else: + logXcpt('addLogFile(%s,%s,%s) IOError' % (sFilename, sDescription, sKind)); + except: + logXcpt('addLogFile(%s,%s,%s)' % (sFilename, sDescription, sKind)); + else: + g_oLock.acquire(); + try: + fRc = g_oReporter.addLogFile(oSrcFile, sFilename, sAltName, sDescription, sKind, sCaller, sTsPrf); + finally: + g_oLock.release(); + oSrcFile.close(); + return fRc; + +def addLogString(sLog, sLogName, sKind, sDescription = ''): + """ + Adds the specified log string to the report. + + The sLog parameter sets the name of the log file. + + The sDescription is a free form description of the log file. + + The sKind parameter is for adding some machine parsable hint what kind of + log file this really is. + + Returns True on success, False on failure (no ENOENT errors are logged). + """ + sTsPrf = utils.getTimePrefix(); + sCaller = utils.getCallerName(); + fRc = False; + + g_oLock.acquire(); + try: + fRc = g_oReporter.addLogString(sLog, sLogName, sDescription, sKind, sCaller, sTsPrf); + finally: + g_oLock.release(); + return fRc; + +def isLocal(): + """Is this a local reporter?""" + return g_oReporter.isLocal() + +def incVerbosity(): + """Increases the verbosity level.""" + return g_oReporter.incVerbosity() + +def incDebug(): + """Increases the debug level.""" + return g_oReporter.incDebug() + +def getVerbosity(): + """Returns the current verbosity level.""" + return g_oReporter.getVerbosity() + +def getDebug(): + """Returns the current debug level.""" + return g_oReporter.getDebug() + +def appendToProcessName(sAppend): + """ + Appends sAppend to the base process name. + Returns the new process name. + """ + return g_oReporter.appendToProcessName(sAppend); + +def getErrorCount(): + """ + Get the current error count for the entire test run. + """ + g_oLock.acquire(); + try: + cErrors = g_oReporter.cErrors; + finally: + g_oLock.release(); + return cErrors; + +def doPollWork(sDebug = None): + """ + This can be called from wait loops and similar to make the reporter call + home with pending XML and such. + """ + g_oReporter.doPollWork(sDebug); + return None; + + +# +# Test reporting, a bit similar to RTTestI*. +# + +def testStart(sName): + """ + Starts a new test (pushes it). + """ + g_oLock.acquire(); + try: + rc = g_oReporter.testStart(sName, utils.getCallerName()); + finally: + g_oLock.release(); + return rc; + +def testValue(sName, sValue, sUnit): + """ + Reports a benchmark value or something simiarlly useful. + """ + g_oLock.acquire(); + try: + rc = g_oReporter.testValue(sName, str(sValue), sUnit, utils.getCallerName()); + finally: + g_oLock.release(); + return rc; + +def testFailure(sDetails): + """ + Reports a failure. + We count these calls and testDone will use them to report PASSED or FAILED. + + Returns False so that a return False line can be saved. + """ + g_oLock.acquire(); + try: + g_oReporter.testFailure(sDetails, utils.getCallerName()); + finally: + g_oLock.release(); + return False; + +def testFailureXcpt(sDetails = ''): + """ + Reports a failure with exception. + We count these calls and testDone will use them to report PASSED or FAILED. + + Returns False so that a return False line can be saved. + """ + # Extract exception info. + try: + oType, oValue, oTraceback = sys.exc_info(); + except: + oType = oValue, oTraceback = None; + if oType is not None: + sCaller = utils.getCallerName(oTraceback.tb_frame); + sXcpt = ' '.join(formatExceptionOnly(oType, oValue, sCaller, utils.getTimePrefix())); + else: + sCaller = utils.getCallerName(); + sXcpt = 'No exception at %s' % (sCaller,); + + # Use testFailure to do the work. + g_oLock.acquire(); + try: + if sDetails == '': + g_oReporter.testFailure('Exception: %s' % (sXcpt,), sCaller); + else: + g_oReporter.testFailure('%s: %s' % (sDetails, sXcpt), sCaller); + finally: + g_oLock.release(); + return False; + +def testDone(fSkipped = False): + """ + Completes the current test (pops it), logging PASSED / FAILURE. + + Returns a tuple with the name of the test and its error count. + """ + g_oLock.acquire(); + try: + rc = g_oReporter.testDone(fSkipped, utils.getCallerName()); + finally: + g_oLock.release(); + return rc; + +def testErrorCount(): + """ + Gets the error count of the current test. + + Returns the number of errors. + """ + g_oLock.acquire(); + try: + cErrors = g_oReporter.testErrorCount(); + finally: + g_oLock.release(); + return cErrors; + +def testCleanup(): + """ + Closes all open tests with a generic error condition. + + Returns True if no open tests, False if something had to be closed with failure. + """ + g_oLock.acquire(); + try: + fRc = g_oReporter.testCleanup(utils.getCallerName()); + g_oReporter.xmlFlush(fRetry = False, fForce = True); + finally: + g_oLock.release(); + fRc = False; + return fRc; + + +# +# Sub XML stuff. +# + +def addSubXmlFile(sFilename): + """ + Adds a sub-xml result file to the party. + """ + fRc = False; + try: + oSrcFile = utils.openNoInherit(sFilename, 'r'); + except IOError as oXcpt: + if oXcpt.errno != errno.ENOENT: + logXcpt('addSubXmlFile(%s)' % (sFilename,)); + except: + logXcpt('addSubXmlFile(%s)' % (sFilename,)); + else: + try: + oWrapper = FileWrapperTestPipe() + oWrapper.write(oSrcFile.read()); + oWrapper.close(); + except: + logXcpt('addSubXmlFile(%s)' % (sFilename,)); + oSrcFile.close(); + + return fRc; + + +# +# Other useful debugging tools. +# + +def logAllStacks(cFrames = None): + """ + Logs the stacks of all python threads. + """ + sTsPrf = utils.getTimePrefix(); + sCaller = utils.getCallerName(); + g_oLock.acquire(); + + cThread = 0; + for idThread, oStack in sys._current_frames().items(): # >=2.5, a bit ugly - pylint: disable=protected-access + try: + if cThread > 0: + g_oReporter.log(1, '', sCaller, sTsPrf); + g_oReporter.log(1, 'Thread %s (%#x)' % (idThread, idThread), sCaller, sTsPrf); + try: + asInfo = traceback.format_stack(oStack, cFrames); + except: + g_oReporter.log(1, ' Stack formatting failed w/ exception', sCaller, sTsPrf); + else: + for sInfo in asInfo: + asLines = sInfo.splitlines(); + for sLine in asLines: + g_oReporter.log(1, sLine, sCaller, sTsPrf); + except: + pass; + cThread += 1; + + g_oLock.release(); + return None; + +def checkTestManagerConnection(): + """ + Checks the connection to the test manager. + + Returns True if the connection is fine, False if not, None if not remote + reporter. + + Note! This as the sideeffect of flushing XML. + """ + g_oLock.acquire(); + try: + fRc = g_oReporter.xmlFlush(fRetry = False, fForce = True); + finally: + g_oLock.release(); + fRc = False; + return fRc; + +def flushall(fSkipXml = False): + """ + Flushes all output streams, both standard and logger related. + This may also push data to the remote test manager. + """ + try: sys.stdout.flush(); + except: pass; + try: sys.stderr.flush(); + except: pass; + + if fSkipXml is not True: + g_oLock.acquire(); + try: + g_oReporter.xmlFlush(fRetry = False); + finally: + g_oLock.release(); + + return True; + + +# +# Module initialization. +# + +def _InitReporterModule(): + """ + Instantiate the test reporter. + """ + global g_oReporter, g_sReporterName + + g_sReporterName = os.getenv("TESTBOX_REPORTER", "local"); + if g_sReporterName == "local": + g_oReporter = LocalReporter(); + elif g_sReporterName == "remote": + g_oReporter = RemoteReporter(); # Correct, but still plain stupid. pylint: disable=redefined-variable-type + else: + print(os.path.basename(__file__) + ": Unknown TESTBOX_REPORTER value: '" + g_sReporterName + "'", file = sys.stderr); + raise Exception("Unknown TESTBOX_REPORTER value '" + g_sReporterName + "'"); + +if __name__ != "checker": # pychecker avoidance. + _InitReporterModule(); diff --git a/src/VBox/ValidationKit/testdriver/testfileset.py b/src/VBox/ValidationKit/testdriver/testfileset.py new file mode 100755 index 00000000..3aa3b0ce --- /dev/null +++ b/src/VBox/ValidationKit/testdriver/testfileset.py @@ -0,0 +1,690 @@ +# -*- coding: utf-8 -*- +# $Id: testfileset.py $ +# pylint: disable=too-many-lines + +""" +Test File Set +""" + +__copyright__ = \ +""" +Copyright (C) 2010-2022 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 154442 $" + + +# Standard Python imports. +import os; +import random; +import string; +import sys; +import tarfile; +import unittest; + +# Validation Kit imports. +from common import utils; +from common import pathutils; +from testdriver import reporter; + +# Python 3 hacks: +if sys.version_info[0] >= 3: + xrange = range; # pylint: disable=redefined-builtin,invalid-name + + + +class TestFsObj(object): + """ A file system object we created in for test purposes. """ + def __init__(self, oParent, sPath, sName = None): + self.oParent = oParent # type: TestDir + self.sPath = sPath # type: str + self.sName = sName # type: str + if oParent: + assert sPath.startswith(oParent.sPath); + assert sName is None; + self.sName = sPath[len(oParent.sPath) + 1:]; + # Add to parent. + oParent.aoChildren.append(self); + oParent.dChildrenUpper[self.sName.upper()] = self; + + def buildPath(self, sRoot, sSep): + """ + Build the path from sRoot using sSep. + + This is handy for getting the path to an object in a different context + (OS, path) than what it was generated for. + """ + if self.oParent: + return self.oParent.buildPath(sRoot, sSep) + sSep + self.sName; + return sRoot + sSep + self.sName; + + +class TestFile(TestFsObj): + """ A file object in the guest. """ + def __init__(self, oParent, sPath, abContent): + TestFsObj.__init__(self, oParent, sPath); + self.abContent = abContent # type: bytearray + self.cbContent = len(abContent); + self.off = 0; + + def read(self, cbToRead): + """ read() emulation. """ + assert self.off <= self.cbContent; + cbLeft = self.cbContent - self.off; + if cbLeft < cbToRead: + cbToRead = cbLeft; + abRet = self.abContent[self.off:(self.off + cbToRead)]; + assert len(abRet) == cbToRead; + self.off += cbToRead; + if sys.version_info[0] < 3: + return bytes(abRet); + return abRet; + + def equalFile(self, oFile): + """ Compares the content of oFile with self.abContent. """ + + # Check the size first. + try: + cbFile = os.fstat(oFile.fileno()).st_size; + except: + return reporter.errorXcpt(); + if cbFile != self.cbContent: + return reporter.error('file size differs: %s, cbContent=%s' % (cbFile, self.cbContent)); + + # Compare the bytes next. + offFile = 0; + try: + oFile.seek(offFile); + except: + return reporter.error('seek error'); + while offFile < self.cbContent: + cbToRead = self.cbContent - offFile; + if cbToRead > 256*1024: + cbToRead = 256*1024; + try: + abRead = oFile.read(cbToRead); + except: + return reporter.error('read error at offset %s' % (offFile,)); + cbRead = len(abRead); + if cbRead == 0: + return reporter.error('premature end of file at offset %s' % (offFile,)); + if not utils.areBytesEqual(abRead, self.abContent[offFile:(offFile + cbRead)]): + return reporter.error('%s byte block at offset %s differs' % (cbRead, offFile,)); + # Advance: + offFile += cbRead; + + return True; + + @staticmethod + def hexFormatBytes(abBuf): + """ Formats a buffer/string/whatever as a string of hex bytes """ + if sys.version_info[0] >= 3: + if utils.isString(abBuf): + try: abBuf = bytes(abBuf, 'utf-8'); + except: pass; + else: + if utils.isString(abBuf): + try: abBuf = bytearray(abBuf, 'utf-8'); # pylint: disable=redefined-variable-type + except: pass; + sRet = ''; + off = 0; + for off, bByte in enumerate(abBuf): + if off > 0: + sRet += ' ' if off & 7 else '-'; + if isinstance(bByte, int): + sRet += '%02x' % (bByte,); + else: + sRet += '%02x' % (ord(bByte),); + return sRet; + + def checkRange(self, cbRange, offFile = 0): + """ Check if the specified range is entirely within the file or not. """ + if offFile >= self.cbContent: + return reporter.error('buffer @ %s LB %s is beyond the end of the file (%s bytes)!' + % (offFile, cbRange, self.cbContent,)); + if offFile + cbRange > self.cbContent: + return reporter.error('buffer @ %s LB %s is partially beyond the end of the file (%s bytes)!' + % (offFile, cbRange, self.cbContent,)); + return True; + + def equalMemory(self, abBuf, offFile = 0): + """ + Compares the content of the given buffer with the file content at that + file offset. + + Returns True if it matches, False + error logging if it does not match. + """ + if not abBuf: + return True; + + if not self.checkRange(len(abBuf), offFile): + return False; + + if sys.version_info[0] >= 3: + if utils.areBytesEqual(abBuf, self.abContent[offFile:(offFile + len(abBuf))]): + return True; + else: + if utils.areBytesEqual(abBuf, buffer(self.abContent, offFile, len(abBuf))): # pylint: disable=undefined-variable + return True; + + reporter.error('mismatch with buffer @ %s LB %s (cbContent=%s)!' % (offFile, len(abBuf), self.cbContent,)); + reporter.error(' type(abBuf): %s' % (type(abBuf),)); + #if isinstance(abBuf, memoryview): + # reporter.error(' nbytes=%s len=%s itemsize=%s type(obj)=%s' + # % (abBuf.nbytes, len(abBuf), abBuf.itemsize, type(abBuf.obj),)); + reporter.error('type(abContent): %s' % (type(self.abContent),)); + + offBuf = 0; + cbLeft = len(abBuf); + while cbLeft > 0: + cbLine = min(16, cbLeft); + abBuf1 = abBuf[offBuf:(offBuf + cbLine)]; + abBuf2 = self.abContent[offFile:(offFile + cbLine)]; + if not utils.areBytesEqual(abBuf1, abBuf2): + try: sStr1 = self.hexFormatBytes(abBuf1); + except: sStr1 = 'oops'; + try: sStr2 = self.hexFormatBytes(abBuf2); + except: sStr2 = 'oops'; + reporter.log('%#10x: %s' % (offBuf, sStr1,)); + reporter.log('%#10x: %s' % (offFile, sStr2,)); + + # Advance. + offBuf += 16; + offFile += 16; + cbLeft -= 16; + + return False; + + +class TestFileZeroFilled(TestFile): + """ + Zero filled test file. + """ + + def __init__(self, oParent, sPath, cbContent): + TestFile.__init__(self, oParent, sPath, bytearray(1)); + self.cbContent = cbContent; + + def read(self, cbToRead): + """ read() emulation. """ + assert self.off <= self.cbContent; + cbLeft = self.cbContent - self.off; + if cbLeft < cbToRead: + cbToRead = cbLeft; + abRet = bytearray(cbToRead); + assert len(abRet) == cbToRead; + self.off += cbToRead; + if sys.version_info[0] < 3: + return bytes(abRet); + return abRet; + + def equalFile(self, oFile): + _ = oFile; + assert False, "not implemented"; + return False; + + def equalMemory(self, abBuf, offFile = 0): + if not abBuf: + return True; + + if not self.checkRange(len(abBuf), offFile): + return False; + + if utils.areBytesEqual(abBuf, bytearray(len(abBuf))): + return True; + + cErrors = 0; + offBuf = 0 + while offBuf < len(abBuf): + bByte = abBuf[offBuf]; + if not isinstance(bByte, int): + bByte = ord(bByte); + if bByte != 0: + reporter.error('Mismatch @ %s/%s: %#x, expected 0!' % (offFile, offBuf, bByte,)); + cErrors += 1; + if cErrors > 32: + return False; + offBuf += 1; + return cErrors == 0; + + +class TestDir(TestFsObj): + """ A file object in the guest. """ + def __init__(self, oParent, sPath, sName = None): + TestFsObj.__init__(self, oParent, sPath, sName); + self.aoChildren = [] # type: list(TestFsObj) + self.dChildrenUpper = {} # type: dict(str, TestFsObj) + + def contains(self, sName): + """ Checks if the directory contains the given name. """ + return sName.upper() in self.dChildrenUpper + + +class TestFileSet(object): + """ + A generated set of files and directories for use in a test. + + Can be wrapped up into a tarball or written directly to the file system. + """ + + ksReservedWinOS2 = '/\\"*:<>?|\t\v\n\r\f\a\b'; + ksReservedUnix = '/'; + ksReservedTrailingWinOS2 = ' .'; + ksReservedTrailingUnix = ''; + + ## @name Path style. + ## @{ + + ## @} + + def __init__(self, fDosStyle, sBasePath, sSubDir, # pylint: disable=too-many-arguments + asCompatibleWith = None, # List of getHostOs values to the names must be compatible with. + oRngFileSizes = xrange(0, 16384), + oRngManyFiles = xrange(128, 512), + oRngTreeFiles = xrange(128, 384), + oRngTreeDepth = xrange(92, 256), + oRngTreeDirs = xrange(2, 16), + cchMaxPath = 230, + cchMaxName = 230, + uSeed = None): + ## @name Parameters + ## @{ + self.fDosStyle = fDosStyle; + self.sMinStyle = 'win' if fDosStyle else 'linux'; + if asCompatibleWith is not None: + for sOs in asCompatibleWith: + assert sOs in ('win', 'os2', 'darwin', 'linux', 'solaris', 'cross'), sOs; + if 'os2' in asCompatibleWith: + self.sMinStyle = 'os2'; + elif 'win' in asCompatibleWith: + self.sMinStyle = 'win'; + # 'cross' marks a lowest common denominator for all supported platforms. + # Used for Guest Control testing. + elif 'cross' in asCompatibleWith: + self.sMinStyle = 'cross'; + self.sBasePath = sBasePath; + self.sSubDir = sSubDir; + self.oRngFileSizes = oRngFileSizes; + self.oRngManyFiles = oRngManyFiles; + self.oRngTreeFiles = oRngTreeFiles; + self.oRngTreeDepth = oRngTreeDepth; + self.oRngTreeDirs = oRngTreeDirs; + self.cchMaxPath = cchMaxPath; + self.cchMaxName = cchMaxName + ## @} + + ## @name Charset stuff + ## @todo allow more chars for unix hosts + guests. + ## @todo include unicode stuff, except on OS/2 and DOS. + ## @{ + ## The filename charset. + self.sFileCharset = string.printable; + ## Set of characters that should not trail a guest filename. + self.sReservedTrailing = self.ksReservedTrailingWinOS2; + if self.sMinStyle in ('win', 'os2'): + for ch in self.ksReservedWinOS2: + self.sFileCharset = self.sFileCharset.replace(ch, ''); + elif self.sMinStyle in ('darwin', 'linux', 'solaris'): + self.sReservedTrailing = self.ksReservedTrailingUnix; + for ch in self.ksReservedUnix: + self.sFileCharset = self.sFileCharset.replace(ch, ''); + else: # 'cross' + # Filter out all reserved charsets from all platforms. + for ch in self.ksReservedWinOS2: + self.sFileCharset = self.sFileCharset.replace(ch, ''); + for ch in self.ksReservedUnix: + self.sFileCharset = self.sFileCharset.replace(ch, ''); + self.sReservedTrailing = self.ksReservedTrailingWinOS2 \ + + self.ksReservedTrailingUnix; + # More spaces and dot: + self.sFileCharset += ' ...'; + ## @} + + ## The root directory. + self.oRoot = None # type: TestDir; + ## An empty directory (under root). + self.oEmptyDir = None # type: TestDir; + + ## A directory with a lot of files in it. + self.oManyDir = None # type: TestDir; + + ## A directory with a mixed tree structure under it. + self.oTreeDir = None # type: TestDir; + ## Number of files in oTreeDir. + self.cTreeFiles = 0; + ## Number of directories under oTreeDir. + self.cTreeDirs = 0; + ## Number of other file types under oTreeDir. + self.cTreeOthers = 0; + + ## All directories in creation order. + self.aoDirs = [] # type: list(TestDir); + ## All files in creation order. + self.aoFiles = [] # type: list(TestFile); + ## Path to object lookup. + self.dPaths = {} # type: dict(str, TestFsObj); + + # + # Do the creating. + # + self.uSeed = uSeed if uSeed is not None else utils.timestampMilli(); + self.oRandom = random.Random(); + self.oRandom.seed(self.uSeed); + reporter.log('prepareGuestForTesting: random seed %s' % (self.uSeed,)); + + self.__createTestStuff(); + + def __createFilename(self, oParent, sCharset, sReservedTrailing): + """ + Creates a filename contains random characters from sCharset and together + with oParent.sPath doesn't exceed the given max chars in length. + """ + ## @todo Consider extending this to take UTF-8 and UTF-16 encoding so we + ## can safely use the full unicode range. Need to check how + ## RTZipTarCmd handles file name encoding in general... + + if oParent: + cchMaxName = self.cchMaxPath - len(oParent.sPath) - 1; + else: + cchMaxName = self.cchMaxPath - 4; + if cchMaxName > self.cchMaxName: + cchMaxName = self.cchMaxName; + if cchMaxName <= 1: + cchMaxName = 2; + + while True: + cchName = self.oRandom.randrange(1, cchMaxName); + sName = ''.join(self.oRandom.choice(sCharset) for _ in xrange(cchName)); + if oParent is None or not oParent.contains(sName): + if sName[-1] not in sReservedTrailing: + if sName not in ('.', '..',): + return sName; + return ''; # never reached, but makes pylint happy. + + def generateFilenameEx(self, cchMax = -1, cchMin = -1): + """ + Generates a filename according to the given specs. + + This is for external use, whereas __createFilename is for internal. + + Returns generated filename. + """ + assert cchMax == -1 or (cchMax >= 1 and cchMax > cchMin); + if cchMin <= 0: + cchMin = 1; + if cchMax < cchMin: + cchMax = self.cchMaxName; + + while True: + cchName = self.oRandom.randrange(cchMin, cchMax + 1); + sName = ''.join(self.oRandom.choice(self.sFileCharset) for _ in xrange(cchName)); + if sName[-1] not in self.sReservedTrailing: + if sName not in ('.', '..',): + return sName; + return ''; # never reached, but makes pylint happy. + + def __createTestDir(self, oParent, sDir, sName = None): + """ + Creates a test directory. + """ + oDir = TestDir(oParent, sDir, sName); + self.aoDirs.append(oDir); + self.dPaths[sDir] = oDir; + return oDir; + + def __createTestFile(self, oParent, sFile): + """ + Creates a test file with random size up to cbMaxContent and random content. + """ + cbFile = self.oRandom.choice(self.oRngFileSizes); + abContent = bytearray(self.oRandom.getrandbits(8) for _ in xrange(cbFile)); + + oFile = TestFile(oParent, sFile, abContent); + self.aoFiles.append(oFile); + self.dPaths[sFile] = oFile; + return oFile; + + def __createTestStuff(self): + """ + Create a random file set that we can work on in the tests. + Returns True/False. + """ + + # + # Create the root test dir. + # + sRoot = pathutils.joinEx(self.fDosStyle, self.sBasePath, self.sSubDir); + self.oRoot = self.__createTestDir(None, sRoot, self.sSubDir); + self.oEmptyDir = self.__createTestDir(self.oRoot, pathutils.joinEx(self.fDosStyle, sRoot, 'empty')); + + # + # Create a directory with lots of files in it: + # + oDir = self.__createTestDir(self.oRoot, pathutils.joinEx(self.fDosStyle, sRoot, 'many')); + self.oManyDir = oDir; + cManyFiles = self.oRandom.choice(self.oRngManyFiles); + for _ in xrange(cManyFiles): + sName = self.__createFilename(oDir, self.sFileCharset, self.sReservedTrailing); + self.__createTestFile(oDir, pathutils.joinEx(self.fDosStyle, oDir.sPath, sName)); + + # + # Generate a tree of files and dirs. + # + oDir = self.__createTestDir(self.oRoot, pathutils.joinEx(self.fDosStyle, sRoot, 'tree')); + uMaxDepth = self.oRandom.choice(self.oRngTreeDepth); + cMaxFiles = self.oRandom.choice(self.oRngTreeFiles); + cMaxDirs = self.oRandom.choice(self.oRngTreeDirs); + self.oTreeDir = oDir; + self.cTreeFiles = 0; + self.cTreeDirs = 0; + uDepth = 0; + while self.cTreeFiles < cMaxFiles and self.cTreeDirs < cMaxDirs: + iAction = self.oRandom.randrange(0, 2+1); + # 0: Add a file: + if iAction == 0 and self.cTreeFiles < cMaxFiles and len(oDir.sPath) < 230 - 2: + sName = self.__createFilename(oDir, self.sFileCharset, self.sReservedTrailing); + self.__createTestFile(oDir, pathutils.joinEx(self.fDosStyle, oDir.sPath, sName)); + self.cTreeFiles += 1; + # 1: Add a subdirector and descend into it: + elif iAction == 1 and self.cTreeDirs < cMaxDirs and uDepth < uMaxDepth and len(oDir.sPath) < 220: + sName = self.__createFilename(oDir, self.sFileCharset, self.sReservedTrailing); + oDir = self.__createTestDir(oDir, pathutils.joinEx(self.fDosStyle, oDir.sPath, sName)); + self.cTreeDirs += 1; + uDepth += 1; + # 2: Ascend to parent dir: + elif iAction == 2 and uDepth > 0: + oDir = oDir.oParent; + uDepth -= 1; + + return True; + + def createTarball(self, sTarFileHst): + """ + Creates a tarball on the host. + Returns success indicator. + """ + reporter.log('Creating tarball "%s" with test files for the guest...' % (sTarFileHst,)); + + cchSkip = len(self.sBasePath) + 1; + + # Open the tarball: + try: + # Make sure to explicitly set GNU_FORMAT here, as with Python 3.8 the default format (tarfile.DEFAULT_FORMAT) + # has been changed to tarfile.PAX_FORMAT, which our extraction code (vts_tar) currently can't handle. + ## @todo Remove tarfile.GNU_FORMAT and use tarfile.PAX_FORMAT as soon as we have PAX support. + oTarFile = tarfile.open(sTarFileHst, 'w:gz', format = tarfile.GNU_FORMAT); # pylint: disable=consider-using-with + except: + return reporter.errorXcpt('Failed to open new tar file: %s' % (sTarFileHst,)); + + # Directories: + for oDir in self.aoDirs: + sPath = oDir.sPath[cchSkip:]; + if self.fDosStyle: + sPath = sPath.replace('\\', '/'); + oTarInfo = tarfile.TarInfo(sPath + '/'); + oTarInfo.mode = 0o777; + oTarInfo.type = tarfile.DIRTYPE; + try: + oTarFile.addfile(oTarInfo); + except: + return reporter.errorXcpt('Failed adding directory tarfile: %s' % (oDir.sPath,)); + + # Files: + for oFile in self.aoFiles: + sPath = oFile.sPath[cchSkip:]; + if self.fDosStyle: + sPath = sPath.replace('\\', '/'); + oTarInfo = tarfile.TarInfo(sPath); + oTarInfo.mode = 0o666; + oTarInfo.size = len(oFile.abContent); + oFile.off = 0; + try: + oTarFile.addfile(oTarInfo, oFile); + except: + return reporter.errorXcpt('Failed adding directory tarfile: %s' % (oFile.sPath,)); + + # Complete the tarball. + try: + oTarFile.close(); + except: + return reporter.errorXcpt('Error closing new tar file: %s' % (sTarFileHst,)); + return True; + + def writeToDisk(self, sAltBase = None): + """ + Writes out the files to disk. + Returns True on success, False + error logging on failure. + """ + + # We only need to flip DOS slashes to unix ones, since windows & OS/2 can handle unix slashes. + fDosToUnix = self.fDosStyle and os.path.sep != '\\'; + + # The directories: + for oDir in self.aoDirs: + sPath = oDir.sPath; + if sAltBase: + if fDosToUnix: + sPath = sAltBase + sPath[len(self.sBasePath):].replace('\\', os.path.sep); + else: + sPath = sAltBase + sPath[len(self.sBasePath):]; + elif fDosToUnix: + sPath = sPath.replace('\\', os.path.sep); + + try: + os.mkdir(sPath, 0o770); + except: + return reporter.errorXcpt('mkdir(%s) failed' % (sPath,)); + + # The files: + for oFile in self.aoFiles: + sPath = oFile.sPath; + if sAltBase: + if fDosToUnix: + sPath = sAltBase + sPath[len(self.sBasePath):].replace('\\', os.path.sep); + else: + sPath = sAltBase + sPath[len(self.sBasePath):]; + elif fDosToUnix: + sPath = sPath.replace('\\', os.path.sep); + + try: + oOutFile = open(sPath, 'wb'); # pylint: disable=consider-using-with + except: + return reporter.errorXcpt('open(%s, "wb") failed' % (sPath,)); + try: + if sys.version_info[0] < 3: + oOutFile.write(bytes(oFile.abContent)); + else: + oOutFile.write(oFile.abContent); + except: + try: oOutFile.close(); + except: pass; + return reporter.errorXcpt('%s: write(%s bytes) failed' % (sPath, oFile.cbContent,)); + try: + oOutFile.close(); + except: + return reporter.errorXcpt('%s: close() failed' % (sPath,)); + + return True; + + + def chooseRandomFile(self): + """ + Returns a random file. + """ + return self.aoFiles[self.oRandom.choice(xrange(len(self.aoFiles)))]; + + def chooseRandomDirFromTree(self, fLeaf = False, fNonEmpty = False, cMaxRetries = 1024): + """ + Returns a random directory from the tree (self.oTreeDir). + Will return None if no directory with given parameters was found. + """ + cRetries = 0; + while cRetries < cMaxRetries: + oDir = self.aoDirs[self.oRandom.choice(xrange(len(self.aoDirs)))]; + # Check fNonEmpty requirement: + if not fNonEmpty or oDir.aoChildren: + # Check leaf requirement: + if not fLeaf: + for oChild in oDir.aoChildren: + if isinstance(oChild, TestDir): + continue; # skip it. + + # Return if in the tree: + oParent = oDir.oParent; + while oParent is not None: + if oParent is self.oTreeDir: + return oDir; + oParent = oParent.oParent; + cRetries += 1; + + return None; # make pylint happy + +# +# Unit testing. +# + +# pylint: disable=missing-docstring +# pylint: disable=undefined-variable +class TestFileSetUnitTests(unittest.TestCase): + def testGeneral(self): + oSet = TestFileSet(False, '/tmp', 'unittest'); + self.assertTrue(isinstance(oSet.chooseRandomDirFromTree(), TestDir)); + self.assertTrue(isinstance(oSet.chooseRandomFile(), TestFile)); + + def testHexFormatBytes(self): + self.assertEqual(TestFile.hexFormatBytes(bytearray([0,1,2,3,4,5,6,7,8,9])), + '00 01 02 03 04 05 06 07-08 09'); + self.assertEqual(TestFile.hexFormatBytes(memoryview(bytearray([0,1,2,3,4,5,6,7,8,9,10, 16]))), + '00 01 02 03 04 05 06 07-08 09 0a 10'); + + +if __name__ == '__main__': + unittest.main(); + # not reached. + diff --git a/src/VBox/ValidationKit/testdriver/tst-txsclient.py b/src/VBox/ValidationKit/testdriver/tst-txsclient.py new file mode 100755 index 00000000..192cf2cb --- /dev/null +++ b/src/VBox/ValidationKit/testdriver/tst-txsclient.py @@ -0,0 +1,315 @@ +# -*- coding: utf-8 -*- +# $Id: tst-txsclient.py $ + +""" +Simple testcase for txsclient.py. +""" + +from __future__ import print_function; + +__copyright__ = \ +""" +Copyright (C) 2010-2022 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 153224 $" + +# Standard python imports. +import os +import sys + +# Validation Kit imports. +sys.path.insert(0, '.'); +sys.path.insert(0, '..'); +from common import utils; +from testdriver import txsclient; +from testdriver import reporter; + +# Python 3 hacks: +if sys.version_info[0] >= 3: + long = int; # pylint: disable=redefined-builtin,invalid-name + +g_cTests = 0; +g_cFailures = 0 + +def boolRes(rc, fExpect = True): + """Checks a boolean result.""" + global g_cTests, g_cFailures; + g_cTests = g_cTests + 1; + if isinstance(rc, bool): + if rc == fExpect: + return 'PASSED'; + g_cFailures = g_cFailures + 1; + return 'FAILED'; + +def stringRes(rc, sExpect): + """Checks a string result.""" + global g_cTests, g_cFailures; + g_cTests = g_cTests + 1; + if utils.isString(rc): + if rc == sExpect: + return 'PASSED'; + g_cFailures = g_cFailures + 1; + return 'FAILED'; + +def main(asArgs): # pylint: disable=missing-docstring,too-many-locals,too-many-statements + cMsTimeout = long(30*1000); + sAddress = 'localhost'; + uPort = None; + fReversedSetup = False; + fReboot = False; + fShutdown = False; + fStdTests = True; + + i = 1; + while i < len(asArgs): + if asArgs[i] == '--hostname': + sAddress = asArgs[i + 1]; + i = i + 2; + elif asArgs[i] == '--port': + uPort = int(asArgs[i + 1]); + i = i + 2; + elif asArgs[i] == '--reversed-setup': + fReversedSetup = True; + i = i + 1; + elif asArgs[i] == '--timeout': + cMsTimeout = long(asArgs[i + 1]); + i = i + 2; + elif asArgs[i] == '--reboot': + fReboot = True; + fShutdown = False; + fStdTests = False; + i = i + 1; + elif asArgs[i] == '--shutdown': + fShutdown = True; + fReboot = False; + fStdTests = False; + i = i + 1; + elif asArgs[i] == '--help': + print('tst-txsclient.py [--hostname <addr|name>] [--port <num>] [--timeout <cMS>] ' + '[--reboot|--shutdown] [--reversed-setup]'); + return 0; + else: + print('Unknown argument: %s' % (asArgs[i])); + return 2; + + if uPort is None: + oSession = txsclient.openTcpSession(cMsTimeout, sAddress, fReversedSetup = fReversedSetup); + else: + oSession = txsclient.openTcpSession(cMsTimeout, sAddress, uPort = uPort, fReversedSetup = fReversedSetup); + if oSession is None: + print('openTcpSession failed'); + return 1; + + fDone = oSession.waitForTask(30*1000); + print('connect: waitForTask -> %s, result %s' % (fDone, oSession.getResult())); + if fDone is True and oSession.isSuccess(): + if fStdTests: + # Get the UUID of the remote instance. + sUuid = oSession.syncUuid(); + if sUuid is not False: + print('%s: UUID = %s' % (boolRes(True), sUuid)); + else: + print('%s: UUID' % (boolRes(False),)); + + # Create and remove a directory on the scratch area. + rc = oSession.syncMkDir('${SCRATCH}/testdir1'); + print('%s: MKDIR(${SCRATCH}/testdir1) -> %s' % (boolRes(rc), rc)); + + rc = oSession.syncIsDir('${SCRATCH}/testdir1'); + print('%s: ISDIR(${SCRATCH}/testdir1) -> %s' % (boolRes(rc), rc)); + + rc = oSession.syncRmDir('${SCRATCH}/testdir1'); + print('%s: RMDIR(${SCRATCH}/testdir1) -> %s' % (boolRes(rc), rc)); + + # Create a two-level subdir. + rc = oSession.syncMkDirPath('${SCRATCH}/testdir2/subdir1'); + print('%s: MKDRPATH(${SCRATCH}/testdir2/subdir1) -> %s' % (boolRes(rc), rc)); + + rc = oSession.syncIsDir('${SCRATCH}/testdir2'); + print('%s: ISDIR(${SCRATCH}/testdir2) -> %s' % (boolRes(rc), rc)); + rc = oSession.syncIsDir('${SCRATCH}/testdir2/'); + print('%s: ISDIR(${SCRATCH}/testdir2/) -> %s' % (boolRes(rc), rc)); + rc = oSession.syncIsDir('${SCRATCH}/testdir2/subdir1'); + print('%s: ISDIR(${SCRATCH}/testdir2/subdir1) -> %s' % (boolRes(rc), rc)); + + rc = oSession.syncRmTree('${SCRATCH}/testdir2'); + print('%s: RMTREE(${SCRATCH}/testdir2) -> %s' % (boolRes(rc), rc)); + + # Check out a simple file. + rc = oSession.syncUploadString('howdy', '${SCRATCH}/howdyfile'); + print('%s: PUT FILE(${SCRATCH}/howdyfile) -> %s' % (boolRes(rc), rc)); + + rc = oSession.syncUploadString('howdy-replaced', '${SCRATCH}/howdyfile'); + print('%s: PUT FILE(${SCRATCH}/howdyfile) -> %s' % (boolRes(rc), rc)); + + rc = oSession.syncDownloadString('${SCRATCH}/howdyfile'); + print('%s: GET FILE(${SCRATCH}/howdyfile) -> "%s" expected "howdy-replaced"' % (stringRes(rc, 'howdy-replaced'), rc)); + + rc = oSession.syncIsFile('${SCRATCH}/howdyfile'); + print('%s: ISFILE(${SCRATCH}/howdyfile) -> %s' % (boolRes(rc), rc)); + rc = oSession.syncIsDir('${SCRATCH}/howdyfile'); + print('%s: ISDIR(${SCRATCH}/howdyfile) -> %s' % (boolRes(rc, False), rc)); + rc = oSession.syncIsSymlink('${SCRATCH}/howdyfile'); + print('%s: ISSYMLNK(${SCRATCH}/howdyfile) -> %s' % (boolRes(rc, False), rc)); + + rc = oSession.syncRmFile('${SCRATCH}/howdyfile'); + print('%s: RMFILE(${SCRATCH}/howdyfile) -> %s' % (boolRes(rc), rc)); + + # Unicode filename (may or may not work, LANG/LC_TYPE dependent on some hosts). + rc = oSession.syncUploadString('howdy', u'${SCRATCH}/Schröder'); + print((u'%s: PUT FILE(${SCRATCH}/Schröder) -> %s' % (boolRes(rc), rc)).encode('ascii', 'replace')); + + rc = oSession.syncIsFile(u'${SCRATCH}/Schröder'); + print((u'%s: ISFILE(${SCRATCH}/Schröder) -> %s' % (boolRes(rc), rc)).encode('ascii', 'replace')); + + rc = oSession.syncRmFile(u'${SCRATCH}/Schröder'); + print((u'%s: RMFILE(${SCRATCH}/Schröder) -> %s' % (boolRes(rc), rc)).encode('ascii', 'replace')); + + # Finally, some file uploading and downloading with unicode filenames. + strUpFile = 'tst-txsclient-upload.bin'; + strDwnFile = 'tst-txsclient-download.bin'; + try: + abRandFile = os.urandom(257897); + except: + print('INFO: no urandom... falling back on a simple string.'); + abRandFile = 'asdflkjasdlfkjasdlfkjq023942relwjgkna9epr865u2nm345;hndafgoukhasre5kb2453km'; + for i in range(1, 64): + abRandFile += abRandFile; + try: + oLocalFile = utils.openNoInherit(strUpFile, 'w+b'); + oLocalFile.write(abRandFile); + oLocalFile.close(); + rc = True; + except: + rc = False; + print('%s: creating file (%s) to upload failed....' % (boolRes(rc), strUpFile)); + + if rc is True: + rc = oSession.syncUploadFile(strUpFile, '${SCRATCH}/tst-txsclient-uploaded.bin') + print('%s: PUT FILE(%s, ${SCRATCH}/tst-txsclient-uploaded.bin) -> %s' % (boolRes(rc), strUpFile, rc)); + + rc = oSession.syncDownloadFile('${SCRATCH}/tst-txsclient-uploaded.bin', strDwnFile) + print('%s: GET FILE(${SCRATCH}/tst-txsclient-uploaded.bin, tst-txsclient-downloaded.txt) -> %s' + % (boolRes(rc), rc)); + + try: + oLocalFile = utils.openNoInherit(strDwnFile, "rb"); + abDwnFile = oLocalFile.read(); + oLocalFile.close(); + if abRandFile == abDwnFile: + print('%s: downloaded file matches the uploaded file' % (boolRes(True),)); + else: + print('%s: downloaded file does not match the uploaded file' % (boolRes(False),)); + print('abRandFile=%s' % (abRandFile,)); + print('abDwnFile =%s' % (abRandFile,)); + except: + print('%s: reading downloaded file (%s) failed....' % (boolRes(False), strDwnFile)); + + rc = oSession.syncRmFile(u'${SCRATCH}/tst-txsclient-uploaded.bin'); + print('%s: RMFILE(${SCRATCH}/tst-txsclient-uploaded.bin) -> %s' % (boolRes(rc), rc)); + + try: os.remove(strUpFile); + except: pass; + try: os.remove(strDwnFile); + except: pass; + + # Execute some simple thing, if available. + # Intentionally skip this test if file is not available due to + # another inserted CD-ROM (e.g. not TestSuite.iso). + sProg = '${CDROM}/${OS/ARCH}/NetPerf${EXESUFF}'; + rc = oSession.syncIsFile(sProg, 30 * 1000, True); + if rc is True: + rc = oSession.syncExecEx(sProg, (sProg, '--help')); + print('%s: EXEC(%s ${SCRATCH}) -> %s' % (boolRes(rc), sProg, rc)); + + rc = oSession.syncExecEx(sProg, (sProg, 'there', 'is no such', 'parameter'), \ + oStdOut='${SCRATCH}/stdout', \ + oStdErr='${SCRATCH}/stderr'); + print('%s: EXEC(%s there is not such parameter > ${SCRATCH}/stdout 2> ${SCRATCH}/stderr) -> %s' + % (boolRes(rc, False), sProg, rc)); + + rc = oSession.syncDownloadString('${SCRATCH}/stdout'); + print('INFO: GET FILE(${SCRATCH}/stdout) -> "%s"' % (rc)); + rc = oSession.syncDownloadString('${SCRATCH}/stderr'); + print('INFO: GET FILE(${SCRATCH}/stderr) -> "%s"' % (rc)); + + print('TESTING: syncExec...'); + rc = oSession.syncExec(sProg, (sProg, '--version')); + print('%s: EXEC(%s --version) -> %s' % (boolRes(rc), sProg, rc)); + + print('TESTING: syncExec...'); + rc = oSession.syncExec(sProg, (sProg, '--help')); + print('%s: EXEC(%s --help) -> %s' % (boolRes(rc), sProg, rc)); + + #print('TESTING: syncExec sleep 30...' + #rc = oSession.syncExec('/usr/bin/sleep', ('/usr/bin/sleep', '30'))); + #print('%s: EXEC(/bin/sleep 30) -> %s' % (boolRes(rc), rc)); + else: + print('SKIP: Execution of %s skipped, does not exist on CD-ROM' % (sProg,)); + + # Execute a non-existing file on CD-ROM. + sProg = '${CDROM}/${OS/ARCH}/NonExisting${EXESUFF}'; + rc = oSession.syncExecEx(sProg, (sProg,), oStdIn = '/dev/null', oStdOut = '/dev/null', \ + oStdErr = '/dev/null', oTestPipe = '/dev/null', \ + sAsUser = '', cMsTimeout = 3600000, fIgnoreErrors = True); + if rc is None: + rc = True; + else: + reporter.error('Unexpected value \"%s\" while executing non-existent file "%s"' % (rc, sProg)); + print('%s: EXEC(%s ${SCRATCH}) -> %s' % (boolRes(rc), sProg, rc)); + + # Done + rc = oSession.syncDisconnect(); + print('%s: disconnect() -> %s' % (boolRes(rc), rc)); + + elif fReboot: + print('TESTING: syncReboot...'); + rc = oSession.syncReboot(); + print('%s: REBOOT() -> %s' % (boolRes(rc), rc)); + elif fShutdown: + print('TESTING: syncShutdown...'); + rc = oSession.syncShutdown(); + print('%s: SHUTDOWN() -> %s' % (boolRes(rc), rc)); + + + if g_cFailures != 0: + print('tst-txsclient.py: %u out of %u test failed' % (g_cFailures, g_cTests)); + return 1; + print('tst-txsclient.py: all %u tests passed!' % (g_cTests)); + return 0; + + +if __name__ == '__main__': + reporter.incVerbosity(); + reporter.incVerbosity(); + reporter.incVerbosity(); + reporter.incVerbosity(); + sys.exit(main(sys.argv)); + diff --git a/src/VBox/ValidationKit/testdriver/txsclient.py b/src/VBox/ValidationKit/testdriver/txsclient.py new file mode 100755 index 00000000..942fab57 --- /dev/null +++ b/src/VBox/ValidationKit/testdriver/txsclient.py @@ -0,0 +1,2376 @@ +# -*- coding: utf-8 -*- +# $Id: txsclient.py $ +# pylint: disable=too-many-lines + +""" +Test eXecution Service Client. +""" +__copyright__ = \ +""" +Copyright (C) 2010-2022 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 154728 $" + +# Standard Python imports. +import array; +import errno; +import os; +import select; +import socket; +import sys; +import threading; +import time; +import zlib; +import uuid; + +# Validation Kit imports. +from common import utils; +from testdriver import base; +from testdriver import reporter; +from testdriver.base import TdTaskBase; + +# Python 3 hacks: +if sys.version_info[0] >= 3: + long = int; # pylint: disable=redefined-builtin,invalid-name + +# +# Helpers for decoding data received from the TXS. +# These are used both the Session and Transport classes. +# + +def getU32(abData, off): + """Get a U32 field.""" + return abData[off] \ + + abData[off + 1] * 256 \ + + abData[off + 2] * 65536 \ + + abData[off + 3] * 16777216; + +def getSZ(abData, off, sDefault = None): + """ + Get a zero-terminated string field. + Returns sDefault if the string is invalid. + """ + cchStr = getSZLen(abData, off); + if cchStr >= 0: + abStr = abData[off:(off + cchStr)]; + try: + if sys.version_info < (3, 9, 0): + # Removed since Python 3.9. + sStr = abStr.tostring(); # pylint: disable=no-member + else: + sStr = abStr.tobytes(); + return sStr.decode('utf_8'); + except: + reporter.errorXcpt('getSZ(,%u)' % (off)); + return sDefault; + +def getSZLen(abData, off): + """ + Get the length of a zero-terminated string field, in bytes. + Returns -1 if off is beyond the data packet or not properly terminated. + """ + cbData = len(abData); + if off >= cbData: + return -1; + + offCur = off; + while abData[offCur] != 0: + offCur = offCur + 1; + if offCur >= cbData: + return -1; + + return offCur - off; + +def isValidOpcodeEncoding(sOpcode): + """ + Checks if the specified opcode is valid or not. + Returns True on success. + Returns False if it is invalid, details in the log. + """ + sSet1 = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + sSet2 = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_ "; + if len(sOpcode) != 8: + reporter.error("invalid opcode length: %s" % (len(sOpcode))); + return False; + for i in range(0, 1): + if sSet1.find(sOpcode[i]) < 0: + reporter.error("invalid opcode char #%u: %s" % (i, sOpcode)); + return False; + for i in range(2, 7): + if sSet2.find(sOpcode[i]) < 0: + reporter.error("invalid opcode char #%u: %s" % (i, sOpcode)); + return False; + return True; + +# +# Helper for encoding data sent to the TXS. +# + +def u32ToByteArray(u32): + """Encodes the u32 value as a little endian byte (B) array.""" + return array.array('B', + ( u32 % 256, + (u32 // 256) % 256, + (u32 // 65536) % 256, + (u32 // 16777216) % 256) ); + +def escapeString(sString): + """ + Does $ escaping of the string so TXS doesn't try do variable expansion. + """ + return sString.replace('$', '$$'); + + + +class TransportBase(object): + """ + Base class for the transport layer. + """ + + def __init__(self, sCaller): + self.sDbgCreated = '%s: %s' % (utils.getTimePrefix(), sCaller); + self.fDummy = 0; + self.abReadAheadHdr = array.array('B'); + + def toString(self): + """ + Stringify the instance for logging and debugging. + """ + return '<%s: abReadAheadHdr=%s, sDbgCreated=%s>' % (type(self).__name__, self.abReadAheadHdr, self.sDbgCreated); + + def __str__(self): + return self.toString(); + + def cancelConnect(self): + """ + Cancels any pending connect() call. + Returns None; + """ + return None; + + def connect(self, cMsTimeout): + """ + Quietly attempts to connect to the TXS. + + Returns True on success. + Returns False on retryable errors (no logging). + Returns None on fatal errors with details in the log. + + Override this method, don't call super. + """ + _ = cMsTimeout; + return False; + + def disconnect(self, fQuiet = False): + """ + Disconnect from the TXS. + + Returns True. + + Override this method, don't call super. + """ + _ = fQuiet; + return True; + + def sendBytes(self, abBuf, cMsTimeout): + """ + Sends the bytes in the buffer abBuf to the TXS. + + Returns True on success. + Returns False on failure and error details in the log. + + Override this method, don't call super. + + Remarks: len(abBuf) is always a multiple of 16. + """ + _ = abBuf; _ = cMsTimeout; + return False; + + def recvBytes(self, cb, cMsTimeout, fNoDataOk): + """ + Receive cb number of bytes from the TXS. + + Returns the bytes (array('B')) on success. + Returns None on failure and error details in the log. + + Override this method, don't call super. + + Remarks: cb is always a multiple of 16. + """ + _ = cb; _ = cMsTimeout; _ = fNoDataOk; + return None; + + def isConnectionOk(self): + """ + Checks if the connection is OK. + + Returns True if it is. + Returns False if it isn't (caller should call diconnect). + + Override this method, don't call super. + """ + return True; + + def isRecvPending(self, cMsTimeout = 0): + """ + Checks if there is incoming bytes, optionally waiting cMsTimeout + milliseconds for something to arrive. + + Returns True if there is, False if there isn't. + + Override this method, don't call super. + """ + _ = cMsTimeout; + return False; + + def sendMsgInt(self, sOpcode, cMsTimeout, abPayload = array.array('B')): + """ + Sends a message (opcode + encoded payload). + + Returns True on success. + Returns False on failure and error details in the log. + """ + # Fix + check the opcode. + if len(sOpcode) < 2: + reporter.fatal('sendMsgInt: invalid opcode length: %d (\"%s\")' % (len(sOpcode), sOpcode)); + return False; + sOpcode = sOpcode.ljust(8); + if not isValidOpcodeEncoding(sOpcode): + reporter.fatal('sendMsgInt: invalid opcode encoding: \"%s\"' % (sOpcode)); + return False; + + # Start construct the message. + cbMsg = 16 + len(abPayload); + abMsg = array.array('B'); + abMsg.extend(u32ToByteArray(cbMsg)); + abMsg.extend((0, 0, 0, 0)); # uCrc32 + try: + abMsg.extend(array.array('B', \ + ( ord(sOpcode[0]), \ + ord(sOpcode[1]), \ + ord(sOpcode[2]), \ + ord(sOpcode[3]), \ + ord(sOpcode[4]), \ + ord(sOpcode[5]), \ + ord(sOpcode[6]), \ + ord(sOpcode[7]) ) ) ); + if abPayload: + abMsg.extend(abPayload); + except: + reporter.fatalXcpt('sendMsgInt: packing problem...'); + return False; + + # checksum it, padd it and send it off. + uCrc32 = zlib.crc32(abMsg[8:]); + abMsg[4:8] = u32ToByteArray(uCrc32); + + while len(abMsg) % 16: + abMsg.append(0); + + reporter.log2('sendMsgInt: op=%s len=%d timeout=%d' % (sOpcode, len(abMsg), cMsTimeout)); + return self.sendBytes(abMsg, cMsTimeout); + + def recvMsg(self, cMsTimeout, fNoDataOk = False): + """ + Receives a message from the TXS. + + Returns the message three-tuple: length, opcode, payload. + Returns (None, None, None) on failure and error details in the log. + """ + + # Read the header. + if self.abReadAheadHdr: + assert(len(self.abReadAheadHdr) == 16); + abHdr = self.abReadAheadHdr; + self.abReadAheadHdr = array.array('B'); + else: + abHdr = self.recvBytes(16, cMsTimeout, fNoDataOk); # (virtual method) # pylint: disable=assignment-from-none + if abHdr is None: + return (None, None, None); + if len(abHdr) != 16: + reporter.fatal('recvBytes(16) returns %d bytes!' % (len(abHdr))); + return (None, None, None); + + # Unpack and validate the header. + cbMsg = getU32(abHdr, 0); + uCrc32 = getU32(abHdr, 4); + + if sys.version_info < (3, 9, 0): + # Removed since Python 3.9. + sOpcode = abHdr[8:16].tostring(); # pylint: disable=no-member + else: + sOpcode = abHdr[8:16].tobytes(); + sOpcode = sOpcode.decode('ascii'); + + if cbMsg < 16: + reporter.fatal('recvMsg: message length is out of range: %s (min 16 bytes)' % (cbMsg)); + return (None, None, None); + if cbMsg > 1024*1024: + reporter.fatal('recvMsg: message length is out of range: %s (max 1MB)' % (cbMsg)); + return (None, None, None); + if not isValidOpcodeEncoding(sOpcode): + reporter.fatal('recvMsg: invalid opcode \"%s\"' % (sOpcode)); + return (None, None, None); + + # Get the payload (if any), dropping the padding. + abPayload = array.array('B'); + if cbMsg > 16: + if cbMsg % 16: + cbPadding = 16 - (cbMsg % 16); + else: + cbPadding = 0; + abPayload = self.recvBytes(cbMsg - 16 + cbPadding, cMsTimeout, False); # pylint: disable=assignment-from-none + if abPayload is None: + self.abReadAheadHdr = abHdr; + if not fNoDataOk : + reporter.log('recvMsg: failed to recv payload bytes!'); + return (None, None, None); + + while cbPadding > 0: + abPayload.pop(); + cbPadding = cbPadding - 1; + + # Check the CRC-32. + if uCrc32 != 0: + uActualCrc32 = zlib.crc32(abHdr[8:]); + if cbMsg > 16: + uActualCrc32 = zlib.crc32(abPayload, uActualCrc32); + uActualCrc32 = uActualCrc32 & 0xffffffff; + if uCrc32 != uActualCrc32: + reporter.fatal('recvMsg: crc error: expected %s, got %s' % (hex(uCrc32), hex(uActualCrc32))); + return (None, None, None); + + reporter.log2('recvMsg: op=%s len=%d' % (sOpcode, len(abPayload))); + return (cbMsg, sOpcode, abPayload); + + def sendMsg(self, sOpcode, cMsTimeout, aoPayload = ()): + """ + Sends a message (opcode + payload tuple). + + Returns True on success. + Returns False on failure and error details in the log. + Returns None if you pass the incorrectly typed parameters. + """ + # Encode the payload. + abPayload = array.array('B'); + for o in aoPayload: + try: + if utils.isString(o): + if sys.version_info[0] >= 3: + abPayload.extend(o.encode('utf_8')); + else: + # the primitive approach... + sUtf8 = o.encode('utf_8'); + for ch in sUtf8: + abPayload.append(ord(ch)) + abPayload.append(0); + elif isinstance(o, (long, int)): + if o < 0 or o > 0xffffffff: + reporter.fatal('sendMsg: uint32_t payload is out of range: %s' % (hex(o))); + return None; + abPayload.extend(u32ToByteArray(o)); + elif isinstance(o, array.array): + abPayload.extend(o); + else: + reporter.fatal('sendMsg: unexpected payload type: %s (%s) (aoPayload=%s)' % (type(o), o, aoPayload)); + return None; + except: + reporter.fatalXcpt('sendMsg: screwed up the encoding code...'); + return None; + return self.sendMsgInt(sOpcode, cMsTimeout, abPayload); + + +class Session(TdTaskBase): + """ + A Test eXecution Service (TXS) client session. + """ + + def __init__(self, oTransport, cMsTimeout, cMsIdleFudge, fTryConnect = False, fnProcessEvents = None): + """ + Construct a TXS session. + + This starts by connecting to the TXS and will enter the signalled state + when connected or the timeout has been reached. + """ + TdTaskBase.__init__(self, utils.getCallerName(), fnProcessEvents); + self.oTransport = oTransport; + self.sStatus = ""; + self.cMsTimeout = 0; + self.fErr = True; # Whether to report errors as error. + self.msStart = 0; + self.oThread = None; + self.fnTask = self.taskDummy; + self.aTaskArgs = None; + self.oTaskRc = None; + self.t3oReply = (None, None, None); + self.fScrewedUpMsgState = False; + self.fTryConnect = fTryConnect; + + if not self.startTask(cMsTimeout, False, "connecting", self.taskConnect, (cMsIdleFudge,)): + raise base.GenError("startTask failed"); + + def __del__(self): + """Make sure to cancel the task when deleted.""" + self.cancelTask(); + + def toString(self): + return '<%s fnTask=%s, aTaskArgs=%s, sStatus=%s, oTaskRc=%s, cMsTimeout=%s,' \ + ' msStart=%s, fTryConnect=%s, fErr=%s, fScrewedUpMsgState=%s, t3oReply=%s oTransport=%s, oThread=%s>' \ + % (TdTaskBase.toString(self), self.fnTask, self.aTaskArgs, self.sStatus, self.oTaskRc, self.cMsTimeout, + self.msStart, self.fTryConnect, self.fErr, self.fScrewedUpMsgState, self.t3oReply, self.oTransport, self.oThread); + + def taskDummy(self): + """Place holder to catch broken state handling.""" + raise Exception(); + + def startTask(self, cMsTimeout, fIgnoreErrors, sStatus, fnTask, aArgs = ()): + """ + Kicks of a new task. + + cMsTimeout: The task timeout in milliseconds. Values less than + 500 ms will be adjusted to 500 ms. This means it is + OK to use negative value. + sStatus: The task status. + fnTask: The method that'll execute the task. + aArgs: Arguments to pass to fnTask. + + Returns True on success, False + error in log on failure. + """ + if not self.cancelTask(): + reporter.maybeErr(not fIgnoreErrors, 'txsclient.Session.startTask: failed to cancel previous task.'); + return False; + + # Change status and make sure we're the + self.lockTask(); + if self.sStatus != "": + self.unlockTask(); + reporter.maybeErr(not fIgnoreErrors, 'txsclient.Session.startTask: race.'); + return False; + self.sStatus = "setup"; + self.oTaskRc = None; + self.t3oReply = (None, None, None); + self.resetTaskLocked(); + self.unlockTask(); + + self.cMsTimeout = max(cMsTimeout, 500); + self.fErr = not fIgnoreErrors; + self.fnTask = fnTask; + self.aTaskArgs = aArgs; + self.oThread = threading.Thread(target=self.taskThread, args=(), name=('TXS-%s' % (sStatus))); + self.oThread.setDaemon(True); # pylint: disable=deprecated-method + self.msStart = base.timestampMilli(); + + self.lockTask(); + self.sStatus = sStatus; + self.unlockTask(); + self.oThread.start(); + + return True; + + def cancelTask(self, fSync = True): + """ + Attempts to cancel any pending tasks. + Returns success indicator (True/False). + """ + self.lockTask(); + + if self.sStatus == "": + self.unlockTask(); + return True; + if self.sStatus == "setup": + self.unlockTask(); + return False; + if self.sStatus == "cancelled": + self.unlockTask(); + return False; + + reporter.log('txsclient: cancelling "%s"...' % (self.sStatus)); + if self.sStatus == 'connecting': + self.oTransport.cancelConnect(); + + self.sStatus = "cancelled"; + oThread = self.oThread; + self.unlockTask(); + + if not fSync: + return False; + + oThread.join(61.0); + + if sys.version_info < (3, 9, 0): + # Removed since Python 3.9. + return oThread.isAlive(); # pylint: disable=no-member + return oThread.is_alive(); + + def taskThread(self): + """ + The task thread function. + This does some housekeeping activities around the real task method call. + """ + if not self.isCancelled(): + try: + fnTask = self.fnTask; + oTaskRc = fnTask(*self.aTaskArgs); + except: + reporter.fatalXcpt('taskThread', 15); + oTaskRc = None; + else: + reporter.log('taskThread: cancelled already'); + + self.lockTask(); + + reporter.log('taskThread: signalling task with status "%s", oTaskRc=%s' % (self.sStatus, oTaskRc)); + self.oTaskRc = oTaskRc; + self.oThread = None; + self.sStatus = ''; + self.signalTaskLocked(); + + self.unlockTask(); + return None; + + def isCancelled(self): + """Internal method for checking if the task has been cancelled.""" + self.lockTask(); + sStatus = self.sStatus; + self.unlockTask(); + if sStatus == "cancelled": + return True; + return False; + + def hasTimedOut(self): + """Internal method for checking if the task has timed out or not.""" + cMsLeft = self.getMsLeft(); + if cMsLeft <= 0: + return True; + return False; + + def getMsLeft(self, cMsMin = 0, cMsMax = -1): + """Gets the time left until the timeout.""" + cMsElapsed = base.timestampMilli() - self.msStart; + if cMsElapsed < 0: + return cMsMin; + cMsLeft = self.cMsTimeout - cMsElapsed; + if cMsLeft <= cMsMin: + return cMsMin; + if cMsLeft > cMsMax > 0: + return cMsMax + return cMsLeft; + + def recvReply(self, cMsTimeout = None, fNoDataOk = False): + """ + Wrapper for TransportBase.recvMsg that stashes the response away + so the client can inspect it later on. + """ + if cMsTimeout is None: + cMsTimeout = self.getMsLeft(500); + cbMsg, sOpcode, abPayload = self.oTransport.recvMsg(cMsTimeout, fNoDataOk); + self.lockTask(); + self.t3oReply = (cbMsg, sOpcode, abPayload); + self.unlockTask(); + return (cbMsg, sOpcode, abPayload); + + def recvAck(self, fNoDataOk = False): + """ + Receives an ACK or error response from the TXS. + + Returns True on success. + Returns False on timeout or transport error. + Returns (sOpcode, sDetails) tuple on failure. The opcode is stripped + and there are always details of some sort or another. + """ + cbMsg, sOpcode, abPayload = self.recvReply(None, fNoDataOk); + if cbMsg is None: + return False; + sOpcode = sOpcode.strip() + if sOpcode == "ACK": + return True; + return (sOpcode, getSZ(abPayload, 0, sOpcode)); + + def recvAckLogged(self, sCommand, fNoDataOk = False): + """ + Wrapper for recvAck and logging. + Returns True on success (ACK). + Returns False on time, transport error and errors signalled by TXS. + """ + rc = self.recvAck(fNoDataOk); + if rc is not True and not fNoDataOk: + if rc is False: + reporter.maybeErr(self.fErr, 'recvAckLogged: %s transport error' % (sCommand)); + else: + reporter.maybeErr(self.fErr, 'recvAckLogged: %s response was %s: %s' % (sCommand, rc[0], rc[1])); + rc = False; + return rc; + + def recvTrueFalse(self, sCommand): + """ + Receives a TRUE/FALSE response from the TXS. + Returns True on TRUE, False on FALSE and None on error/other (logged). + """ + cbMsg, sOpcode, abPayload = self.recvReply(); + if cbMsg is None: + reporter.maybeErr(self.fErr, 'recvAckLogged: %s transport error' % (sCommand)); + return None; + + sOpcode = sOpcode.strip() + if sOpcode == "TRUE": + return True; + if sOpcode == "FALSE": + return False; + reporter.maybeErr(self.fErr, 'recvAckLogged: %s response was %s: %s' % (sCommand, sOpcode, getSZ(abPayload, 0, sOpcode))); + return None; + + def sendMsg(self, sOpcode, aoPayload = (), cMsTimeout = None): + """ + Wrapper for TransportBase.sendMsg that inserts the correct timeout. + """ + if cMsTimeout is None: + cMsTimeout = self.getMsLeft(500); + return self.oTransport.sendMsg(sOpcode, cMsTimeout, aoPayload); + + def asyncToSync(self, fnAsync, *aArgs): + """ + Wraps an asynchronous task into a synchronous operation. + + Returns False on failure, task return status on success. + """ + rc = fnAsync(*aArgs); + if rc is False: + reporter.log2('asyncToSync(%s): returns False (#1)' % (fnAsync)); + return rc; + + rc = self.waitForTask(self.cMsTimeout + 5000); + if rc is False: + reporter.maybeErr(self.fErr, 'asyncToSync: waitForTask (timeout %d) failed...' % (self.cMsTimeout,)); + self.cancelTask(); + #reporter.log2('asyncToSync(%s): returns False (#2)' % (fnAsync, rc)); + return False; + + rc = self.getResult(); + #reporter.log2('asyncToSync(%s): returns %s' % (fnAsync, rc)); + return rc; + + # + # Connection tasks. + # + + def taskConnect(self, cMsIdleFudge): + """Tries to connect to the TXS""" + while not self.isCancelled(): + reporter.log2('taskConnect: connecting ...'); + rc = self.oTransport.connect(self.getMsLeft(500)); + if rc is True: + reporter.log('taskConnect: succeeded'); + return self.taskGreet(cMsIdleFudge); + if rc is None: + reporter.log2('taskConnect: unable to connect'); + return None; + if self.hasTimedOut(): + reporter.log2('taskConnect: timed out'); + if not self.fTryConnect: + reporter.maybeErr(self.fErr, 'taskConnect: timed out'); + return False; + time.sleep(self.getMsLeft(1, 1000) / 1000.0); + if not self.fTryConnect: + reporter.maybeErr(self.fErr, 'taskConnect: cancelled'); + return False; + + def taskGreet(self, cMsIdleFudge): + """Greets the TXS""" + rc = self.sendMsg("HOWDY", ()); + if rc is True: + rc = self.recvAckLogged("HOWDY", self.fTryConnect); + if rc is True: + while cMsIdleFudge > 0: + cMsIdleFudge -= 1000; + time.sleep(1); + else: + self.oTransport.disconnect(self.fTryConnect); + return rc; + + def taskBye(self): + """Says goodbye to the TXS""" + rc = self.sendMsg("BYE"); + if rc is True: + rc = self.recvAckLogged("BYE"); + self.oTransport.disconnect(); + return rc; + + def taskVer(self): + """Requests version information from TXS""" + rc = self.sendMsg("VER"); + if rc is True: + rc = False; + cbMsg, sOpcode, abPayload = self.recvReply(); + if cbMsg is not None: + sOpcode = sOpcode.strip(); + if sOpcode == "ACK VER": + sVer = getSZ(abPayload, 0); + if sVer is not None: + rc = sVer; + else: + reporter.maybeErr(self.fErr, 'taskVer got a bad reply: %s' % (sOpcode,)); + else: + reporter.maybeErr(self.fErr, 'taskVer got 3xNone from recvReply.'); + return rc; + + def taskUuid(self): + """Gets the TXS UUID""" + rc = self.sendMsg("UUID"); + if rc is True: + rc = False; + cbMsg, sOpcode, abPayload = self.recvReply(); + if cbMsg is not None: + sOpcode = sOpcode.strip() + if sOpcode == "ACK UUID": + sUuid = getSZ(abPayload, 0); + if sUuid is not None: + sUuid = '{%s}' % (sUuid,) + try: + _ = uuid.UUID(sUuid); + rc = sUuid; + except: + reporter.errorXcpt('taskUuid got an invalid UUID string %s' % (sUuid,)); + else: + reporter.maybeErr(self.fErr, 'taskUuid did not get a UUID string.'); + else: + reporter.maybeErr(self.fErr, 'taskUuid got a bad reply: %s' % (sOpcode,)); + else: + reporter.maybeErr(self.fErr, 'taskUuid got 3xNone from recvReply.'); + return rc; + + # + # Process task + # pylint: disable=missing-docstring + # + + def taskExecEx(self, sExecName, fFlags, asArgs, asAddEnv, oStdIn, oStdOut, oStdErr, oTestPipe, sAsUser): # pylint: disable=too-many-arguments,too-many-locals,too-many-statements,line-too-long + # Construct the payload. + aoPayload = [long(fFlags), '%s' % (sExecName), long(len(asArgs))]; + for sArg in asArgs: + aoPayload.append('%s' % (sArg)); + aoPayload.append(long(len(asAddEnv))); + for sPutEnv in asAddEnv: + aoPayload.append('%s' % (sPutEnv)); + for o in (oStdIn, oStdOut, oStdErr, oTestPipe): + if utils.isString(o): + aoPayload.append(o); + elif o is not None: + aoPayload.append('|'); + o.uTxsClientCrc32 = zlib.crc32(b''); + else: + aoPayload.append(''); + aoPayload.append('%s' % (sAsUser)); + aoPayload.append(long(self.cMsTimeout)); + + # Kick of the EXEC command. + rc = self.sendMsg('EXEC', aoPayload) + if rc is True: + rc = self.recvAckLogged('EXEC'); + if rc is True: + # Loop till the process completes, feed input to the TXS and + # receive output from it. + sFailure = ""; + msPendingInputReply = None; + cbMsg, sOpcode, abPayload = (None, None, None); + while True: + # Pending input? + if msPendingInputReply is None \ + and oStdIn is not None \ + and not utils.isString(oStdIn): + try: + sInput = oStdIn.read(65536); + except: + reporter.errorXcpt('read standard in'); + sFailure = 'exception reading stdin'; + rc = None; + break; + if sInput: + # Convert to a byte array before handing it of to sendMsg or the string + # will get some zero termination added breaking the CRC (and injecting + # unwanted bytes). + abInput = array.array('B', sInput.encode('utf-8')); + oStdIn.uTxsClientCrc32 = zlib.crc32(abInput, oStdIn.uTxsClientCrc32); + rc = self.sendMsg('STDIN', (long(oStdIn.uTxsClientCrc32 & 0xffffffff), abInput)); + if rc is not True: + sFailure = 'sendMsg failure'; + break; + msPendingInputReply = base.timestampMilli(); + continue; + + rc = self.sendMsg('STDINEOS'); + oStdIn = None; + if rc is not True: + sFailure = 'sendMsg failure'; + break; + msPendingInputReply = base.timestampMilli(); + + # Wait for input (500 ms timeout). + if cbMsg is None: + cbMsg, sOpcode, abPayload = self.recvReply(cMsTimeout=500, fNoDataOk=True); + if cbMsg is None: + # Check for time out before restarting the loop. + # Note! Only doing timeout checking here does mean that + # the TXS may prevent us from timing out by + # flooding us with data. This is unlikely though. + if self.hasTimedOut() \ + and ( msPendingInputReply is None \ + or base.timestampMilli() - msPendingInputReply > 30000): + reporter.maybeErr(self.fErr, 'taskExecEx: timed out'); + sFailure = 'timeout'; + rc = None; + break; + # Check that the connection is OK. + if not self.oTransport.isConnectionOk(): + self.oTransport.disconnect(); + sFailure = 'disconnected'; + rc = False; + break; + continue; + + # Handle the response. + sOpcode = sOpcode.rstrip(); + if sOpcode == 'STDOUT': + oOut = oStdOut; + elif sOpcode == 'STDERR': + oOut = oStdErr; + elif sOpcode == 'TESTPIPE': + oOut = oTestPipe; + else: + oOut = None; + if oOut is not None: + # Output from the process. + if len(abPayload) < 4: + sFailure = 'malformed output packet (%s, %u bytes)' % (sOpcode, cbMsg); + reporter.maybeErr(self.fErr, 'taskExecEx: %s' % (sFailure)); + rc = None; + break; + uStreamCrc32 = getU32(abPayload, 0); + oOut.uTxsClientCrc32 = zlib.crc32(abPayload[4:], oOut.uTxsClientCrc32); + if uStreamCrc32 != (oOut.uTxsClientCrc32 & 0xffffffff): + sFailure = 'crc error - mine=%#x their=%#x (%s, %u bytes)' \ + % (oOut.uTxsClientCrc32 & 0xffffffff, uStreamCrc32, sOpcode, cbMsg); + reporter.maybeErr(self.fErr, 'taskExecEx: %s' % (sFailure)); + rc = None; + break; + try: + oOut.write(abPayload[4:]); + except: + sFailure = 'exception writing %s' % (sOpcode); + reporter.errorXcpt('taskExecEx: %s' % (sFailure)); + rc = None; + break; + elif sOpcode == 'STDINIGN' and msPendingInputReply is not None: + # Standard input is ignored. Ignore this condition for now. + msPendingInputReply = None; + reporter.log('taskExecEx: Standard input is ignored... why?'); + del oStdIn.uTxsClientCrc32; + oStdIn = '/dev/null'; + elif sOpcode in ('STDINMEM', 'STDINBAD', 'STDINCRC',)\ + and msPendingInputReply is not None: + # TXS STDIN error, abort. + # TODO: STDINMEM - consider undoing the previous stdin read and try resubmitt it. + msPendingInputReply = None; + sFailure = 'TXS is out of memory for std input buffering'; + reporter.maybeErr(self.fErr, 'taskExecEx: %s' % (sFailure)); + rc = None; + break; + elif sOpcode == 'ACK' and msPendingInputReply is not None: + msPendingInputReply = None; + elif sOpcode.startswith('PROC '): + # Process status message, handle it outside the loop. + rc = True; + break; + else: + sFailure = 'Unexpected opcode %s' % (sOpcode); + reporter.maybeErr(self.fErr, 'taskExecEx: %s' % (sFailure)); + rc = None; + break; + # Clear the message. + cbMsg, sOpcode, abPayload = (None, None, None); + + # If we sent an STDIN packet and didn't get a reply yet, we'll give + # TXS some 5 seconds to reply to this. If we don't wait here we'll + # get screwed later on if we mix it up with the reply to some other + # command. Hackish. + if msPendingInputReply is not None: + cbMsg2, sOpcode2, abPayload2 = self.oTransport.recvMsg(5000); + if cbMsg2 is not None: + reporter.log('taskExecEx: Out of order STDIN, got reply: %s, %s, %s [ignored]' + % (cbMsg2, sOpcode2, abPayload2)); + msPendingInputReply = None; + else: + reporter.maybeErr(self.fErr, 'taskExecEx: Pending STDIN, no reply after 5 secs!'); + self.fScrewedUpMsgState = True; + + # Parse the exit status (True), abort (None) or do nothing (False). + if rc is True: + if sOpcode == 'PROC OK': + pass; + else: + rc = False; + # Do proper parsing some other day if needed: + # PROC TOK, PROC TOA, PROC DWN, PROC DOO, + # PROC NOK + rc, PROC SIG + sig, PROC ABD, FAILED. + if sOpcode == 'PROC DOO': + reporter.log('taskExecEx: PROC DOO[FUS]: %s' % (abPayload,)); + elif sOpcode.startswith('PROC NOK'): + reporter.log('taskExecEx: PROC NOK: rcExit=%s' % (abPayload,)); + elif abPayload and sOpcode.startswith('PROC '): + reporter.log('taskExecEx: %s payload=%s' % (sOpcode, abPayload,)); + + else: + if rc is None: + # Abort it. + reporter.log('taskExecEx: sending ABORT...'); + rc = self.sendMsg('ABORT'); + while rc is True: + cbMsg, sOpcode, abPayload = self.oTransport.recvMsg(30000); + if cbMsg is None: + reporter.maybeErr(self.fErr, 'taskExecEx: Pending ABORT, no reply after 30 secs!') + self.fScrewedUpMsgState = True; + break; + if sOpcode.startswith('PROC '): + reporter.log('taskExecEx: ABORT reply: %s, %s, %s [ignored]' % (cbMsg, sOpcode, abPayload)); + break; + reporter.log('taskExecEx: ABORT in process, ignoring reply: %s, %s, %s' % (cbMsg, sOpcode, abPayload)); + # Check that the connection is OK before looping. + if not self.oTransport.isConnectionOk(): + self.oTransport.disconnect(); + break; + + # Fake response with the reason why we quit. + if sFailure is not None: + self.t3oReply = (0, 'EXECFAIL', sFailure); + rc = None; + else: + rc = None; + + # Cleanup. + for o in (oStdIn, oStdOut, oStdErr, oTestPipe): + if o is not None and not utils.isString(o): + del o.uTxsClientCrc32; # pylint: disable=maybe-no-member + # Make sure all files are closed + o.close(); # pylint: disable=maybe-no-member + reporter.log('taskExecEx: returns %s' % (rc)); + return rc; + + # + # Admin tasks + # + + def hlpRebootShutdownWaitForAck(self, sCmd): + """Wait for reboot/shutodwn ACK.""" + rc = self.recvAckLogged(sCmd); + if rc is True: + # poll a little while for server to disconnect. + uMsStart = base.timestampMilli(); + while self.oTransport.isConnectionOk() \ + and base.timestampMilli() - uMsStart >= 5000: + if self.oTransport.isRecvPending(min(500, self.getMsLeft())): + break; + self.oTransport.disconnect(); + return rc; + + def taskReboot(self): + rc = self.sendMsg('REBOOT'); + if rc is True: + rc = self.hlpRebootShutdownWaitForAck('REBOOT'); + return rc; + + def taskShutdown(self): + rc = self.sendMsg('SHUTDOWN'); + if rc is True: + rc = self.hlpRebootShutdownWaitForAck('SHUTDOWN'); + return rc; + + # + # CD/DVD control tasks. + # + + ## TODO + + # + # File system tasks + # + + def taskMkDir(self, sRemoteDir, fMode): + rc = self.sendMsg('MKDIR', (fMode, sRemoteDir)); + if rc is True: + rc = self.recvAckLogged('MKDIR'); + return rc; + + def taskMkDirPath(self, sRemoteDir, fMode): + rc = self.sendMsg('MKDRPATH', (fMode, sRemoteDir)); + if rc is True: + rc = self.recvAckLogged('MKDRPATH'); + return rc; + + def taskMkSymlink(self, sLinkTarget, sLink): + rc = self.sendMsg('MKSYMLNK', (sLinkTarget, sLink)); + if rc is True: + rc = self.recvAckLogged('MKSYMLNK'); + return rc; + + def taskRmDir(self, sRemoteDir): + rc = self.sendMsg('RMDIR', (sRemoteDir,)); + if rc is True: + rc = self.recvAckLogged('RMDIR'); + return rc; + + def taskRmFile(self, sRemoteFile): + rc = self.sendMsg('RMFILE', (sRemoteFile,)); + if rc is True: + rc = self.recvAckLogged('RMFILE'); + return rc; + + def taskRmSymlink(self, sRemoteSymlink): + rc = self.sendMsg('RMSYMLNK', (sRemoteSymlink,)); + if rc is True: + rc = self.recvAckLogged('RMSYMLNK'); + return rc; + + def taskRmTree(self, sRemoteTree): + rc = self.sendMsg('RMTREE', (sRemoteTree,)); + if rc is True: + rc = self.recvAckLogged('RMTREE'); + return rc; + + def taskChMod(self, sRemotePath, fMode): + rc = self.sendMsg('CHMOD', (int(fMode), sRemotePath,)); + if rc is True: + rc = self.recvAckLogged('CHMOD'); + return rc; + + def taskChOwn(self, sRemotePath, idUser, idGroup): + rc = self.sendMsg('CHOWN', (int(idUser), int(idGroup), sRemotePath,)); + if rc is True: + rc = self.recvAckLogged('CHOWN'); + return rc; + + def taskIsDir(self, sRemoteDir): + rc = self.sendMsg('ISDIR', (sRemoteDir,)); + if rc is True: + rc = self.recvTrueFalse('ISDIR'); + return rc; + + def taskIsFile(self, sRemoteFile): + rc = self.sendMsg('ISFILE', (sRemoteFile,)); + if rc is True: + rc = self.recvTrueFalse('ISFILE'); + return rc; + + def taskIsSymlink(self, sRemoteSymlink): + rc = self.sendMsg('ISSYMLNK', (sRemoteSymlink,)); + if rc is True: + rc = self.recvTrueFalse('ISSYMLNK'); + return rc; + + #def "STAT " + #def "LSTAT " + #def "LIST " + + def taskCopyFile(self, sSrcFile, sDstFile, fMode, fFallbackOkay): + """ Copies a file within the remote from source to destination. """ + _ = fFallbackOkay; # Not used yet. + # Note: If fMode is set to 0, it's up to the target OS' implementation with + # what a file mode the destination file gets created (i.e. via umask). + rc = self.sendMsg('CPFILE', (int(fMode), sSrcFile, sDstFile,)); + if rc is True: + rc = self.recvAckLogged('CPFILE'); + return rc; + + def taskUploadFile(self, sLocalFile, sRemoteFile, fMode, fFallbackOkay): + # + # Open the local file (make sure it exist before bothering TXS) and + # tell TXS that we want to upload a file. + # + try: + oLocalFile = utils.openNoInherit(sLocalFile, 'rb'); + except: + reporter.errorXcpt('taskUpload: failed to open "%s"' % (sLocalFile)); + return False; + + # Common cause with taskUploadStr + rc = self.taskUploadCommon(oLocalFile, sRemoteFile, fMode, fFallbackOkay); + + # Cleanup. + oLocalFile.close(); + return rc; + + def taskUploadString(self, sContent, sRemoteFile, fMode, fFallbackOkay): + # Wrap sContent in a file like class. + class InStringFile(object): # pylint: disable=too-few-public-methods + def __init__(self, sContent): + self.sContent = sContent; + self.off = 0; + + def read(self, cbMax): + cbLeft = len(self.sContent) - self.off; + if cbLeft == 0: + return ""; + if cbLeft <= cbMax: + sRet = self.sContent[self.off:(self.off + cbLeft)]; + else: + sRet = self.sContent[self.off:(self.off + cbMax)]; + self.off = self.off + len(sRet); + return sRet; + + oLocalString = InStringFile(sContent); + return self.taskUploadCommon(oLocalString, sRemoteFile, fMode, fFallbackOkay); + + def taskUploadCommon(self, oLocalFile, sRemoteFile, fMode, fFallbackOkay): + """Common worker used by taskUploadFile and taskUploadString.""" + # + # Command + ACK. + # + # Only used the new PUT2FILE command if we've got a non-zero mode mask. + # Fall back on the old command if the new one is not known by the TXS. + # + if fMode == 0: + rc = self.sendMsg('PUT FILE', (sRemoteFile,)); + if rc is True: + rc = self.recvAckLogged('PUT FILE'); + else: + rc = self.sendMsg('PUT2FILE', (fMode, sRemoteFile)); + if rc is True: + rc = self.recvAck(); + if rc is False: + reporter.maybeErr(self.fErr, 'recvAckLogged: PUT2FILE transport error'); + elif rc is not True: + if rc[0] == 'UNKNOWN' and fFallbackOkay: + # Fallback: + rc = self.sendMsg('PUT FILE', (sRemoteFile,)); + if rc is True: + rc = self.recvAckLogged('PUT FILE'); + else: + reporter.maybeErr(self.fErr, 'recvAckLogged: PUT2FILE response was %s: %s' % (rc[0], rc[1],)); + rc = False; + if rc is True: + # + # Push data packets until eof. + # + uMyCrc32 = zlib.crc32(b''); + while True: + # Read up to 64 KB of data. + try: + sRaw = oLocalFile.read(65536); + except: + rc = None; + break; + + # Convert to array - this is silly! + abBuf = array.array('B'); + if utils.isString(sRaw): + for i, _ in enumerate(sRaw): + abBuf.append(ord(sRaw[i])); + else: + abBuf.extend(sRaw); + sRaw = None; + + # Update the file stream CRC and send it off. + uMyCrc32 = zlib.crc32(abBuf, uMyCrc32); + if not abBuf: + rc = self.sendMsg('DATA EOF', (long(uMyCrc32 & 0xffffffff), )); + else: + rc = self.sendMsg('DATA ', (long(uMyCrc32 & 0xffffffff), abBuf)); + if rc is False: + break; + + # Wait for the reply. + rc = self.recvAck(); + if rc is not True: + if rc is False: + reporter.maybeErr(self.fErr, 'taskUpload: transport error waiting for ACK'); + else: + reporter.maybeErr(self.fErr, 'taskUpload: DATA response was %s: %s' % (rc[0], rc[1])); + rc = False; + break; + + # EOF? + if not abBuf: + break; + + # Send ABORT on ACK and I/O errors. + if rc is None: + rc = self.sendMsg('ABORT'); + if rc is True: + self.recvAckLogged('ABORT'); + rc = False; + return rc; + + def taskDownloadFile(self, sRemoteFile, sLocalFile): + try: + oLocalFile = utils.openNoInherit(sLocalFile, 'wb'); + except: + reporter.errorXcpt('taskDownload: failed to open "%s"' % (sLocalFile)); + return False; + + rc = self.taskDownloadCommon(sRemoteFile, oLocalFile); + + oLocalFile.close(); + if rc is False: + try: + os.remove(sLocalFile); + except: + reporter.errorXcpt(); + return rc; + + def taskDownloadString(self, sRemoteFile, sEncoding = 'utf-8', fIgnoreEncodingErrors = True): + # Wrap sContent in a file like class. + class OutStringFile(object): # pylint: disable=too-few-public-methods + def __init__(self): + self.asContent = []; + + def write(self, sBuf): + self.asContent.append(sBuf); + return None; + + oLocalString = OutStringFile(); + rc = self.taskDownloadCommon(sRemoteFile, oLocalString); + if rc is True: + rc = ''; + for sBuf in oLocalString.asContent: + if hasattr(sBuf, 'decode'): + rc += sBuf.decode(sEncoding, 'ignore' if fIgnoreEncodingErrors else 'strict'); + else: + rc += sBuf; + return rc; + + def taskDownloadCommon(self, sRemoteFile, oLocalFile): + """Common worker for taskDownloadFile and taskDownloadString.""" + rc = self.sendMsg('GET FILE', (sRemoteFile,)) + if rc is True: + # + # Process data packets until eof. + # + uMyCrc32 = zlib.crc32(b''); + while rc is True: + cbMsg, sOpcode, abPayload = self.recvReply(); + if cbMsg is None: + reporter.maybeErr(self.fErr, 'taskDownload got 3xNone from recvReply.'); + rc = None; + break; + + # Validate. + sOpcode = sOpcode.rstrip(); + if sOpcode not in ('DATA', 'DATA EOF',): + reporter.maybeErr(self.fErr, 'taskDownload got a error reply: opcode="%s" details="%s"' + % (sOpcode, getSZ(abPayload, 0, "None"))); + rc = False; + break; + if sOpcode == 'DATA' and len(abPayload) < 4: + reporter.maybeErr(self.fErr, 'taskDownload got a bad DATA packet: len=%u' % (len(abPayload))); + rc = None; + break; + if sOpcode == 'DATA EOF' and len(abPayload) != 4: + reporter.maybeErr(self.fErr, 'taskDownload got a bad EOF packet: len=%u' % (len(abPayload))); + rc = None; + break; + + # Check the CRC (common for both packets). + uCrc32 = getU32(abPayload, 0); + if sOpcode == 'DATA': + uMyCrc32 = zlib.crc32(abPayload[4:], uMyCrc32); + if uCrc32 != (uMyCrc32 & 0xffffffff): + reporter.maybeErr(self.fErr, 'taskDownload got a bad CRC: mycrc=%s remotecrc=%s' + % (hex(uMyCrc32), hex(uCrc32))); + rc = None; + break; + if sOpcode == 'DATA EOF': + rc = self.sendMsg('ACK'); + break; + + # Finally, push the data to the file. + try: + if sys.version_info < (3, 9, 0): + # Removed since Python 3.9. + abData = abPayload[4:].tostring(); + else: + abData = abPayload[4:].tobytes(); + oLocalFile.write(abData); + except: + reporter.errorXcpt('I/O error writing to "%s"' % (sRemoteFile)); + rc = None; + break; + rc = self.sendMsg('ACK'); + + # Send NACK on validation and I/O errors. + if rc is None: + rc = self.sendMsg('NACK'); + rc = False; + return rc; + + def taskPackFile(self, sRemoteFile, sRemoteSource): + rc = self.sendMsg('PKFILE', (sRemoteFile, sRemoteSource)); + if rc is True: + rc = self.recvAckLogged('PKFILE'); + return rc; + + def taskUnpackFile(self, sRemoteFile, sRemoteDir): + rc = self.sendMsg('UNPKFILE', (sRemoteFile, sRemoteDir)); + if rc is True: + rc = self.recvAckLogged('UNPKFILE'); + return rc; + + def taskExpandString(self, sString): + rc = self.sendMsg('EXP STR ', (sString,)); + if rc is True: + rc = False; + cbMsg, sOpcode, abPayload = self.recvReply(); + if cbMsg is not None: + sOpcode = sOpcode.strip(); + if sOpcode == "STRING": + sStringExp = getSZ(abPayload, 0); + if sStringExp is not None: + rc = sStringExp; + else: # Also handles SHORTSTR reply (not enough space to store result). + reporter.maybeErr(self.fErr, 'taskExpandString got a bad reply: %s' % (sOpcode,)); + else: + reporter.maybeErr(self.fErr, 'taskExpandString got 3xNone from recvReply.'); + return rc; + + # pylint: enable=missing-docstring + + + # + # Public methods - generic task queries + # + + def isSuccess(self): + """Returns True if the task completed successfully, otherwise False.""" + self.lockTask(); + sStatus = self.sStatus; + oTaskRc = self.oTaskRc; + self.unlockTask(); + if sStatus != "": + return False; + if oTaskRc is False or oTaskRc is None: + return False; + return True; + + def getResult(self): + """ + Returns the result of a completed task. + Returns None if not completed yet or no previous task. + """ + self.lockTask(); + sStatus = self.sStatus; + oTaskRc = self.oTaskRc; + self.unlockTask(); + if sStatus != "": + return None; + return oTaskRc; + + def getLastReply(self): + """ + Returns the last reply three-tuple: cbMsg, sOpcode, abPayload. + Returns a None, None, None three-tuple if there was no last reply. + """ + self.lockTask(); + t3oReply = self.t3oReply; + self.unlockTask(); + return t3oReply; + + # + # Public methods - connection. + # + + def asyncDisconnect(self, cMsTimeout = 30000, fIgnoreErrors = False): + """ + Initiates a disconnect task. + + Returns True on success, False on failure (logged). + + The task returns True on success and False on failure. + """ + return self.startTask(cMsTimeout, fIgnoreErrors, "bye", self.taskBye); + + def syncDisconnect(self, cMsTimeout = 30000, fIgnoreErrors = False): + """Synchronous version.""" + return self.asyncToSync(self.asyncDisconnect, cMsTimeout, fIgnoreErrors); + + def asyncVer(self, cMsTimeout = 30000, fIgnoreErrors = False): + """ + Initiates a task for getting the TXS version information. + + Returns True on success, False on failure (logged). + + The task returns the version string on success and False on failure. + """ + return self.startTask(cMsTimeout, fIgnoreErrors, "ver", self.taskVer); + + def syncVer(self, cMsTimeout = 30000, fIgnoreErrors = False): + """Synchronous version.""" + return self.asyncToSync(self.asyncVer, cMsTimeout, fIgnoreErrors); + + def asyncUuid(self, cMsTimeout = 30000, fIgnoreErrors = False): + """ + Initiates a task for getting the TXS UUID. + + Returns True on success, False on failure (logged). + + The task returns UUID string (in {}) on success and False on failure. + """ + return self.startTask(cMsTimeout, fIgnoreErrors, "uuid", self.taskUuid); + + def syncUuid(self, cMsTimeout = 30000, fIgnoreErrors = False): + """Synchronous version.""" + return self.asyncToSync(self.asyncUuid, cMsTimeout, fIgnoreErrors); + + # + # Public methods - execution. + # + + def asyncExecEx(self, sExecName, asArgs = (), asAddEnv = (), # pylint: disable=too-many-arguments + oStdIn = None, oStdOut = None, oStdErr = None, oTestPipe = None, + sAsUser = "", cMsTimeout = 3600000, fIgnoreErrors = False): + """ + Initiates a exec process task. + + Returns True on success, False on failure (logged). + + The task returns True if the process exited normally with status code 0. + The task returns None if on failure prior to executing the process, and + False if the process exited with a different status or in an abnormal + manner. Both None and False are logged of course and further info can + also be obtained by getLastReply(). + + The oStdIn, oStdOut, oStdErr and oTestPipe specifiy how to deal with + these streams. If None, no special action is taken and the output goes + to where ever the TXS sends its output, and ditto for input. + - To send to / read from the bitbucket, pass '/dev/null'. + - To redirect to/from a file, just specify the remote filename. + - To append to a file use '>>' followed by the remote filename. + - To pipe the stream to/from the TXS, specify a file like + object. For StdIn a non-blocking read() method is required. For + the other a write() method is required. Watch out for deadlock + conditions between StdIn and StdOut/StdErr/TestPipe piping. + """ + return self.startTask(cMsTimeout, fIgnoreErrors, "exec", self.taskExecEx, + (sExecName, long(0), asArgs, asAddEnv, oStdIn, + oStdOut, oStdErr, oTestPipe, sAsUser)); + + def syncExecEx(self, sExecName, asArgs = (), asAddEnv = (), # pylint: disable=too-many-arguments + oStdIn = '/dev/null', oStdOut = '/dev/null', + oStdErr = '/dev/null', oTestPipe = '/dev/null', + sAsUser = '', cMsTimeout = 3600000, fIgnoreErrors = False): + """Synchronous version.""" + return self.asyncToSync(self.asyncExecEx, sExecName, asArgs, asAddEnv, oStdIn, oStdOut, \ + oStdErr, oTestPipe, sAsUser, cMsTimeout, fIgnoreErrors); + + def asyncExec(self, sExecName, asArgs = (), asAddEnv = (), sAsUser = "", fWithTestPipe = True, sPrefix = '', \ + cMsTimeout = 3600000, fIgnoreErrors = False): + """ + Initiates a exec process test task. + + Returns True on success, False on failure (logged). + + The task returns True if the process exited normally with status code 0. + The task returns None if on failure prior to executing the process, and + False if the process exited with a different status or in an abnormal + manner. Both None and False are logged of course and further info can + also be obtained by getLastReply(). + + Standard in is taken from /dev/null. While both standard output and + standard error goes directly to reporter.log(). The testpipe is piped + to reporter.xxxx. + """ + + sStdIn = '/dev/null'; + oStdOut = reporter.FileWrapper('%sstdout' % sPrefix); + oStdErr = reporter.FileWrapper('%sstderr' % sPrefix); + if fWithTestPipe: oTestPipe = reporter.FileWrapperTestPipe(); + else: oTestPipe = '/dev/null'; # pylint: disable=redefined-variable-type + + return self.startTask(cMsTimeout, fIgnoreErrors, "exec", self.taskExecEx, + (sExecName, long(0), asArgs, asAddEnv, sStdIn, oStdOut, oStdErr, oTestPipe, sAsUser)); + + def syncExec(self, sExecName, asArgs = (), asAddEnv = (), sAsUser = '', fWithTestPipe = True, sPrefix = '', + cMsTimeout = 3600000, fIgnoreErrors = False): + """Synchronous version.""" + return self.asyncToSync(self.asyncExec, sExecName, asArgs, asAddEnv, sAsUser, fWithTestPipe, sPrefix, \ + cMsTimeout, fIgnoreErrors); + + # + # Public methods - system + # + + def asyncReboot(self, cMsTimeout = 30000, fIgnoreErrors = False): + """ + Initiates a reboot task. + + Returns True on success, False on failure (logged). + + The task returns True on success, False on failure (logged). The + session will be disconnected on successful task completion. + """ + return self.startTask(cMsTimeout, fIgnoreErrors, "reboot", self.taskReboot, ()); + + def syncReboot(self, cMsTimeout = 30000, fIgnoreErrors = False): + """Synchronous version.""" + return self.asyncToSync(self.asyncReboot, cMsTimeout, fIgnoreErrors); + + def asyncShutdown(self, cMsTimeout = 30000, fIgnoreErrors = False): + """ + Initiates a shutdown task. + + Returns True on success, False on failure (logged). + + The task returns True on success, False on failure (logged). + """ + return self.startTask(cMsTimeout, fIgnoreErrors, "shutdown", self.taskShutdown, ()); + + def syncShutdown(self, cMsTimeout = 30000, fIgnoreErrors = False): + """Synchronous version.""" + return self.asyncToSync(self.asyncShutdown, cMsTimeout, fIgnoreErrors); + + + # + # Public methods - file system + # + + def asyncMkDir(self, sRemoteDir, fMode = 0o700, cMsTimeout = 30000, fIgnoreErrors = False): + """ + Initiates a mkdir task. + + Returns True on success, False on failure (logged). + + The task returns True on success, False on failure (logged). + """ + return self.startTask(cMsTimeout, fIgnoreErrors, "mkDir", self.taskMkDir, (sRemoteDir, long(fMode))); + + def syncMkDir(self, sRemoteDir, fMode = 0o700, cMsTimeout = 30000, fIgnoreErrors = False): + """Synchronous version.""" + return self.asyncToSync(self.asyncMkDir, sRemoteDir, long(fMode), cMsTimeout, fIgnoreErrors); + + def asyncMkDirPath(self, sRemoteDir, fMode = 0o700, cMsTimeout = 30000, fIgnoreErrors = False): + """ + Initiates a mkdir -p task. + + Returns True on success, False on failure (logged). + + The task returns True on success, False on failure (logged). + """ + return self.startTask(cMsTimeout, fIgnoreErrors, "mkDirPath", self.taskMkDirPath, (sRemoteDir, long(fMode))); + + def syncMkDirPath(self, sRemoteDir, fMode = 0o700, cMsTimeout = 30000, fIgnoreErrors = False): + """Synchronous version.""" + return self.asyncToSync(self.asyncMkDirPath, sRemoteDir, long(fMode), cMsTimeout, fIgnoreErrors); + + def asyncMkSymlink(self, sLinkTarget, sLink, cMsTimeout = 30000, fIgnoreErrors = False): + """ + Initiates a symlink task. + + Returns True on success, False on failure (logged). + + The task returns True on success, False on failure (logged). + """ + return self.startTask(cMsTimeout, fIgnoreErrors, "mkSymlink", self.taskMkSymlink, (sLinkTarget, sLink)); + + def syncMkSymlink(self, sLinkTarget, sLink, cMsTimeout = 30000, fIgnoreErrors = False): + """Synchronous version.""" + return self.asyncToSync(self.asyncMkSymlink, sLinkTarget, sLink, cMsTimeout, fIgnoreErrors); + + def asyncRmDir(self, sRemoteDir, cMsTimeout = 30000, fIgnoreErrors = False): + """ + Initiates a rmdir task. + + Returns True on success, False on failure (logged). + + The task returns True on success, False on failure (logged). + """ + return self.startTask(cMsTimeout, fIgnoreErrors, "rmDir", self.taskRmDir, (sRemoteDir,)); + + def syncRmDir(self, sRemoteDir, cMsTimeout = 30000, fIgnoreErrors = False): + """Synchronous version.""" + return self.asyncToSync(self.asyncRmDir, sRemoteDir, cMsTimeout, fIgnoreErrors); + + def asyncRmFile(self, sRemoteFile, cMsTimeout = 30000, fIgnoreErrors = False): + """ + Initiates a rmfile task. + + Returns True on success, False on failure (logged). + + The task returns True on success, False on failure (logged). + """ + return self.startTask(cMsTimeout, fIgnoreErrors, "rmFile", self.taskRmFile, (sRemoteFile,)); + + def syncRmFile(self, sRemoteFile, cMsTimeout = 30000, fIgnoreErrors = False): + """Synchronous version.""" + return self.asyncToSync(self.asyncRmFile, sRemoteFile, cMsTimeout, fIgnoreErrors); + + def asyncRmSymlink(self, sRemoteSymlink, cMsTimeout = 30000, fIgnoreErrors = False): + """ + Initiates a rmsymlink task. + + Returns True on success, False on failure (logged). + + The task returns True on success, False on failure (logged). + """ + return self.startTask(cMsTimeout, fIgnoreErrors, "rmSymlink", self.taskRmSymlink, (sRemoteSymlink,)); + + def syncRmSymlink(self, sRemoteSymlink, cMsTimeout = 30000, fIgnoreErrors = False): + """Synchronous version.""" + return self.asyncToSync(self.asyncRmSymlink, sRemoteSymlink, cMsTimeout, fIgnoreErrors); + + def asyncRmTree(self, sRemoteTree, cMsTimeout = 30000, fIgnoreErrors = False): + """ + Initiates a rmtree task. + + Returns True on success, False on failure (logged). + + The task returns True on success, False on failure (logged). + """ + return self.startTask(cMsTimeout, fIgnoreErrors, "rmTree", self.taskRmTree, (sRemoteTree,)); + + def syncRmTree(self, sRemoteTree, cMsTimeout = 30000, fIgnoreErrors = False): + """Synchronous version.""" + return self.asyncToSync(self.asyncRmTree, sRemoteTree, cMsTimeout, fIgnoreErrors); + + def asyncChMod(self, sRemotePath, fMode, cMsTimeout = 30000, fIgnoreErrors = False): + """ + Initiates a chmod task. + + Returns True on success, False on failure (logged). + + The task returns True on success, False on failure (logged). + """ + return self.startTask(cMsTimeout, fIgnoreErrors, "chMod", self.taskChMod, (sRemotePath, fMode)); + + def syncChMod(self, sRemotePath, fMode, cMsTimeout = 30000, fIgnoreErrors = False): + """Synchronous version.""" + return self.asyncToSync(self.asyncChMod, sRemotePath, fMode, cMsTimeout, fIgnoreErrors); + + def asyncChOwn(self, sRemotePath, idUser, idGroup, cMsTimeout = 30000, fIgnoreErrors = False): + """ + Initiates a chown task. + + Returns True on success, False on failure (logged). + + The task returns True on success, False on failure (logged). + """ + return self.startTask(cMsTimeout, fIgnoreErrors, "chOwn", self.taskChOwn, (sRemotePath, idUser, idGroup)); + + def syncChOwn(self, sRemotePath, idUser, idGroup, cMsTimeout = 30000, fIgnoreErrors = False): + """Synchronous version.""" + return self.asyncToSync(self.asyncChMod, sRemotePath, idUser, idGroup, cMsTimeout, fIgnoreErrors); + + def asyncIsDir(self, sRemoteDir, cMsTimeout = 30000, fIgnoreErrors = False): + """ + Initiates a is-dir query task. + + Returns True on success, False on failure (logged). + + The task returns True if it's a directory, False if it isn't, and + None on error (logged). + """ + return self.startTask(cMsTimeout, fIgnoreErrors, "isDir", self.taskIsDir, (sRemoteDir,)); + + def syncIsDir(self, sRemoteDir, cMsTimeout = 30000, fIgnoreErrors = False): + """Synchronous version.""" + return self.asyncToSync(self.asyncIsDir, sRemoteDir, cMsTimeout, fIgnoreErrors); + + def asyncIsFile(self, sRemoteFile, cMsTimeout = 30000, fIgnoreErrors = False): + """ + Initiates a is-file query task. + + Returns True on success, False on failure (logged). + + The task returns True if it's a file, False if it isn't, and None on + error (logged). + """ + return self.startTask(cMsTimeout, fIgnoreErrors, "isFile", self.taskIsFile, (sRemoteFile,)); + + def syncIsFile(self, sRemoteFile, cMsTimeout = 30000, fIgnoreErrors = False): + """Synchronous version.""" + return self.asyncToSync(self.asyncIsFile, sRemoteFile, cMsTimeout, fIgnoreErrors); + + def asyncIsSymlink(self, sRemoteSymlink, cMsTimeout = 30000, fIgnoreErrors = False): + """ + Initiates a is-symbolic-link query task. + + Returns True on success, False on failure (logged). + + The task returns True if it's a symbolic linke, False if it isn't, and + None on error (logged). + """ + return self.startTask(cMsTimeout, fIgnoreErrors, "isSymlink", self.taskIsSymlink, (sRemoteSymlink,)); + + def syncIsSymlink(self, sRemoteSymlink, cMsTimeout = 30000, fIgnoreErrors = False): + """Synchronous version.""" + return self.asyncToSync(self.asyncIsSymlink, sRemoteSymlink, cMsTimeout, fIgnoreErrors); + + #def "STAT " + #def "LSTAT " + #def "LIST " + + @staticmethod + def calcFileXferTimeout(cbFile): + """ + Calculates a reasonable timeout for an upload/download given the file size. + + Returns timeout in milliseconds. + """ + return 30000 + cbFile / 32; # 32 KiB/s (picked out of thin air) + + @staticmethod + def calcUploadTimeout(sLocalFile): + """ + Calculates a reasonable timeout for an upload given the file (will stat it). + + Returns timeout in milliseconds. + """ + try: cbFile = os.path.getsize(sLocalFile); + except: cbFile = 1024*1024; + return Session.calcFileXferTimeout(cbFile); + + def asyncCopyFile(self, sSrcFile, sDstFile, + fMode = 0, fFallbackOkay = True, cMsTimeout = 30000, fIgnoreErrors = False): + """ + Initiates a file copying task on the remote. + + Returns True on success, False on failure (logged). + + The task returns True on success, False on failure (logged). + """ + return self.startTask(cMsTimeout, fIgnoreErrors, "cpfile", + self.taskCopyFile, (sSrcFile, sDstFile, fMode, fFallbackOkay)); + + def syncCopyFile(self, sSrcFile, sDstFile, fMode = 0, cMsTimeout = 30000, fIgnoreErrors = False): + """Synchronous version.""" + return self.asyncToSync(self.asyncCopyFile, sSrcFile, sDstFile, fMode, cMsTimeout, fIgnoreErrors); + + def asyncUploadFile(self, sLocalFile, sRemoteFile, + fMode = 0, fFallbackOkay = True, cMsTimeout = 30000, fIgnoreErrors = False): + """ + Initiates a download query task. + + Returns True on success, False on failure (logged). + + The task returns True on success, False on failure (logged). + """ + return self.startTask(cMsTimeout, fIgnoreErrors, "upload", + self.taskUploadFile, (sLocalFile, sRemoteFile, fMode, fFallbackOkay)); + + def syncUploadFile(self, sLocalFile, sRemoteFile, fMode = 0, fFallbackOkay = True, cMsTimeout = 0, fIgnoreErrors = False): + """Synchronous version.""" + if cMsTimeout <= 0: + cMsTimeout = self.calcUploadTimeout(sLocalFile); + return self.asyncToSync(self.asyncUploadFile, sLocalFile, sRemoteFile, fMode, fFallbackOkay, cMsTimeout, fIgnoreErrors); + + def asyncUploadString(self, sContent, sRemoteFile, + fMode = 0, fFallbackOkay = True, cMsTimeout = 0, fIgnoreErrors = False): + """ + Initiates a upload string task. + + Returns True on success, False on failure (logged). + + The task returns True on success, False on failure (logged). + """ + if cMsTimeout <= 0: + cMsTimeout = self.calcFileXferTimeout(len(sContent)); + return self.startTask(cMsTimeout, fIgnoreErrors, "uploadString", + self.taskUploadString, (sContent, sRemoteFile, fMode, fFallbackOkay)); + + def syncUploadString(self, sContent, sRemoteFile, fMode = 0, fFallbackOkay = True, cMsTimeout = 0, fIgnoreErrors = False): + """Synchronous version.""" + if cMsTimeout <= 0: + cMsTimeout = self.calcFileXferTimeout(len(sContent)); + return self.asyncToSync(self.asyncUploadString, sContent, sRemoteFile, fMode, fFallbackOkay, cMsTimeout, fIgnoreErrors); + + def asyncDownloadFile(self, sRemoteFile, sLocalFile, cMsTimeout = 120000, fIgnoreErrors = False): + """ + Initiates a download file task. + + Returns True on success, False on failure (logged). + + The task returns True on success, False on failure (logged). + """ + return self.startTask(cMsTimeout, fIgnoreErrors, "downloadFile", self.taskDownloadFile, (sRemoteFile, sLocalFile)); + + def syncDownloadFile(self, sRemoteFile, sLocalFile, cMsTimeout = 120000, fIgnoreErrors = False): + """Synchronous version.""" + return self.asyncToSync(self.asyncDownloadFile, sRemoteFile, sLocalFile, cMsTimeout, fIgnoreErrors); + + def asyncDownloadString(self, sRemoteFile, sEncoding = 'utf-8', fIgnoreEncodingErrors = True, + cMsTimeout = 30000, fIgnoreErrors = False): + """ + Initiates a download string task. + + Returns True on success, False on failure (logged). + + The task returns a byte string on success, False on failure (logged). + """ + return self.startTask(cMsTimeout, fIgnoreErrors, "downloadString", + self.taskDownloadString, (sRemoteFile, sEncoding, fIgnoreEncodingErrors)); + + def syncDownloadString(self, sRemoteFile, sEncoding = 'utf-8', fIgnoreEncodingErrors = True, + cMsTimeout = 30000, fIgnoreErrors = False): + """Synchronous version.""" + return self.asyncToSync(self.asyncDownloadString, sRemoteFile, sEncoding, fIgnoreEncodingErrors, + cMsTimeout, fIgnoreErrors); + + def asyncPackFile(self, sRemoteFile, sRemoteSource, cMsTimeout = 120000, fIgnoreErrors = False): + """ + Initiates a packing file/directory task. + + Returns True on success, False on failure (logged). + + The task returns True on success, False on failure (logged). + """ + return self.startTask(cMsTimeout, fIgnoreErrors, "packFile", self.taskPackFile, + (sRemoteFile, sRemoteSource)); + + def syncPackFile(self, sRemoteFile, sRemoteSource, cMsTimeout = 120000, fIgnoreErrors = False): + """Synchronous version.""" + return self.asyncToSync(self.asyncPackFile, sRemoteFile, sRemoteSource, cMsTimeout, fIgnoreErrors); + + def asyncUnpackFile(self, sRemoteFile, sRemoteDir, cMsTimeout = 120000, fIgnoreErrors = False): + """ + Initiates a unpack file task. + + Returns True on success, False on failure (logged). + + The task returns True on success, False on failure (logged). + """ + return self.startTask(cMsTimeout, fIgnoreErrors, "unpackFile", self.taskUnpackFile, + (sRemoteFile, sRemoteDir)); + + def syncUnpackFile(self, sRemoteFile, sRemoteDir, cMsTimeout = 120000, fIgnoreErrors = False): + """Synchronous version.""" + return self.asyncToSync(self.asyncUnpackFile, sRemoteFile, sRemoteDir, cMsTimeout, fIgnoreErrors); + + def asyncExpandString(self, sString, cMsTimeout = 120000, fIgnoreErrors = False): + """ + Initiates an expand string task. + + Returns expanded string on success, False on failure (logged). + + The task returns True on success, False on failure (logged). + """ + return self.startTask(cMsTimeout, fIgnoreErrors, "expandString", + self.taskExpandString, (sString,)); + + def syncExpandString(self, sString, cMsTimeout = 120000, fIgnoreErrors = False): + """Synchronous version.""" + return self.asyncToSync(self.asyncExpandString, sString, cMsTimeout, fIgnoreErrors); + + +class TransportTcp(TransportBase): + """ + TCP transport layer for the TXS client session class. + """ + + def __init__(self, sHostname, uPort, fReversedSetup): + """ + Save the parameters. The session will call us back to make the + connection later on its worker thread. + """ + TransportBase.__init__(self, utils.getCallerName()); + self.sHostname = sHostname; + self.fReversedSetup = fReversedSetup; + self.uPort = uPort if uPort is not None else 5042 if fReversedSetup is False else 5048; + self.oSocket = None; + self.oWakeupW = None; + self.oWakeupR = None; + self.fConnectCanceled = False; + self.fIsConnecting = False; + self.oCv = threading.Condition(); + self.abReadAhead = array.array('B'); + + def toString(self): + return '<%s sHostname=%s, fReversedSetup=%s, uPort=%s, oSocket=%s,'\ + ' fConnectCanceled=%s, fIsConnecting=%s, oCv=%s, abReadAhead=%s>' \ + % (TransportBase.toString(self), self.sHostname, self.fReversedSetup, self.uPort, self.oSocket, + self.fConnectCanceled, self.fIsConnecting, self.oCv, self.abReadAhead); + + def __isInProgressXcpt(self, oXcpt): + """ In progress exception? """ + try: + if isinstance(oXcpt, socket.error): + try: + if oXcpt.errno == errno.EINPROGRESS: + return True; + except: pass; + # Windows? + try: + if oXcpt.errno == errno.EWOULDBLOCK: + return True; + except: pass; + except: + pass; + return False; + + def __isWouldBlockXcpt(self, oXcpt): + """ Would block exception? """ + try: + if isinstance(oXcpt, socket.error): + try: + if oXcpt.errno == errno.EWOULDBLOCK: + return True; + except: pass; + try: + if oXcpt.errno == errno.EAGAIN: + return True; + except: pass; + except: + pass; + return False; + + def __isConnectionReset(self, oXcpt): + """ Connection reset by Peer or others. """ + try: + if isinstance(oXcpt, socket.error): + try: + if oXcpt.errno == errno.ECONNRESET: + return True; + except: pass; + try: + if oXcpt.errno == errno.ENETRESET: + return True; + except: pass; + except: + pass; + return False; + + def _closeWakeupSockets(self): + """ Closes the wakup sockets. Caller should own the CV. """ + oWakeupR = self.oWakeupR; + self.oWakeupR = None; + if oWakeupR is not None: + oWakeupR.close(); + + oWakeupW = self.oWakeupW; + self.oWakeupW = None; + if oWakeupW is not None: + oWakeupW.close(); + + return None; + + def cancelConnect(self): + # This is bad stuff. + self.oCv.acquire(); + reporter.log2('TransportTcp::cancelConnect: fIsConnecting=%s oSocket=%s' % (self.fIsConnecting, self.oSocket)); + self.fConnectCanceled = True; + if self.fIsConnecting: + oSocket = self.oSocket; + self.oSocket = None; + if oSocket is not None: + reporter.log2('TransportTcp::cancelConnect: closing the socket'); + oSocket.close(); + + oWakeupW = self.oWakeupW; + self.oWakeupW = None; + if oWakeupW is not None: + reporter.log2('TransportTcp::cancelConnect: wakeup call'); + try: oWakeupW.send(b'cancelled!\n'); + except: reporter.logXcpt(); + try: oWakeupW.shutdown(socket.SHUT_WR); + except: reporter.logXcpt(); + oWakeupW.close(); + self.oCv.release(); + + def _connectAsServer(self, oSocket, oWakeupR, cMsTimeout): + """ Connects to the TXS server as server, i.e. the reversed setup. """ + assert(self.fReversedSetup); + + reporter.log2('TransportTcp::_connectAsServer: oSocket=%s, cMsTimeout=%u' % (oSocket, cMsTimeout)); + + # Workaround for bind() failure... + try: + oSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1); + except: + reporter.errorXcpt('socket.listen(1) failed'); + return None; + + # Bind the socket and make it listen. + try: + oSocket.bind((self.sHostname, self.uPort)); + except: + reporter.errorXcpt('socket.bind((%s,%s)) failed' % (self.sHostname, self.uPort)); + return None; + try: + oSocket.listen(1); + except: + reporter.errorXcpt('socket.listen(1) failed'); + return None; + + # Accept connections. + oClientSocket = None; + tClientAddr = None; + try: + (oClientSocket, tClientAddr) = oSocket.accept(); + except socket.error as e: + if not self.__isInProgressXcpt(e): + raise; + + # Do the actual waiting. + reporter.log2('TransportTcp::accept: operation in progress (%s)...' % (e,)); + try: + select.select([oSocket, oWakeupR], [], [oSocket, oWakeupR], cMsTimeout / 1000.0); + except socket.error as oXctp: + if oXctp.errno != errno.EBADF or not self.fConnectCanceled: + raise; + reporter.log('socket.select() on accept was canceled'); + return None; + except: + reporter.logXcpt('socket.select() on accept'); + + # Try accept again. + try: + (oClientSocket, tClientAddr) = oSocket.accept(); + except socket.error as oXcpt: + if not self.__isInProgressXcpt(e): + if oXcpt.errno != errno.EBADF or not self.fConnectCanceled: + raise; + reporter.log('socket.accept() was canceled'); + return None; + reporter.log('socket.accept() timed out'); + return False; + except: + reporter.errorXcpt('socket.accept() failed'); + return None; + except: + reporter.errorXcpt('socket.accept() failed'); + return None; + + # Store the connected socket and throw away the server socket. + self.oCv.acquire(); + if not self.fConnectCanceled: + self.oSocket.close(); + self.oSocket = oClientSocket; + self.sHostname = "%s:%s" % (tClientAddr[0], tClientAddr[1]); + self.oCv.release(); + return True; + + def _connectAsClient(self, oSocket, oWakeupR, cMsTimeout): + """ Connects to the TXS server as client. """ + assert(not self.fReversedSetup); + + # Connect w/ timeouts. + rc = None; + try: + oSocket.connect((self.sHostname, self.uPort)); + rc = True; + except socket.error as oXcpt: + iRc = oXcpt.errno; + if self.__isInProgressXcpt(oXcpt): + # Do the actual waiting. + reporter.log2('TransportTcp::connect: operation in progress (%s)...' % (oXcpt,)); + try: + ttRc = select.select([oWakeupR], [oSocket], [oSocket, oWakeupR], cMsTimeout / 1000.0); + if len(ttRc[1]) + len(ttRc[2]) == 0: + raise socket.error(errno.ETIMEDOUT, 'select timed out'); + iRc = oSocket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR); + rc = iRc == 0; + except socket.error as oXcpt2: + iRc = oXcpt2.errno; + except: + iRc = -42; + reporter.fatalXcpt('socket.select() on connect failed'); + + if rc is True: + pass; + elif iRc in (errno.ECONNREFUSED, errno.EHOSTUNREACH, errno.EINTR, errno.ENETDOWN, errno.ENETUNREACH, errno.ETIMEDOUT): + rc = False; # try again. + else: + if iRc != errno.EBADF or not self.fConnectCanceled: + reporter.fatalXcpt('socket.connect((%s,%s)) failed; iRc=%s' % (self.sHostname, self.uPort, iRc)); + reporter.log2('TransportTcp::connect: rc=%s iRc=%s' % (rc, iRc)); + except: + reporter.fatalXcpt('socket.connect((%s,%s)) failed' % (self.sHostname, self.uPort)); + return rc; + + + def connect(self, cMsTimeout): + # Create a non-blocking socket. + reporter.log2('TransportTcp::connect: cMsTimeout=%s sHostname=%s uPort=%s' % (cMsTimeout, self.sHostname, self.uPort)); + try: + oSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0); + except: + reporter.fatalXcpt('socket.socket() failed'); + return None; + try: + oSocket.setblocking(0); + except: + oSocket.close(); + reporter.fatalXcpt('socket.socket() failed'); + return None; + + # Create wakeup socket pair for unix (select doesn't wake up on socket close on Linux). + oWakeupR = None; + oWakeupW = None; + if hasattr(socket, 'socketpair'): + try: (oWakeupR, oWakeupW) = socket.socketpair(); # pylint: disable=no-member + except: reporter.logXcpt('socket.socketpair() failed'); + + # Update the state. + self.oCv.acquire(); + rc = None; + if not self.fConnectCanceled: + self.oSocket = oSocket; + self.oWakeupW = oWakeupW; + self.oWakeupR = oWakeupR; + self.fIsConnecting = True; + self.oCv.release(); + + # Try connect. + if oWakeupR is None: + oWakeupR = oSocket; # Avoid select failure. + if self.fReversedSetup: + rc = self._connectAsServer(oSocket, oWakeupR, cMsTimeout); + else: + rc = self._connectAsClient(oSocket, oWakeupR, cMsTimeout); + oSocket = None; + + # Update the state and cleanup on failure/cancel. + self.oCv.acquire(); + if rc is True and self.fConnectCanceled: + rc = False; + self.fIsConnecting = False; + + if rc is not True: + if self.oSocket is not None: + self.oSocket.close(); + self.oSocket = None; + self._closeWakeupSockets(); + self.oCv.release(); + + reporter.log2('TransportTcp::connect: returning %s' % (rc,)); + return rc; + + def disconnect(self, fQuiet = False): + if self.oSocket is not None: + self.abReadAhead = array.array('B'); + + # Try a shutting down the socket gracefully (draining it). + try: + self.oSocket.shutdown(socket.SHUT_WR); + except: + if not fQuiet: + reporter.error('shutdown(SHUT_WR)'); + try: + self.oSocket.setblocking(0); # just in case it's not set. + sData = "1"; + while sData: + sData = self.oSocket.recv(16384); + except: + pass; + + # Close it. + self.oCv.acquire(); + try: self.oSocket.setblocking(1); + except: pass; + self.oSocket.close(); + self.oSocket = None; + else: + self.oCv.acquire(); + self._closeWakeupSockets(); + self.oCv.release(); + + def sendBytes(self, abBuf, cMsTimeout): + if self.oSocket is None: + reporter.error('TransportTcp.sendBytes: No connection.'); + return False; + + # Try send it all. + try: + cbSent = self.oSocket.send(abBuf); + if cbSent == len(abBuf): + return True; + except Exception as oXcpt: + if not self.__isWouldBlockXcpt(oXcpt): + reporter.errorXcpt('TranportTcp.sendBytes: %s bytes' % (len(abBuf))); + return False; + cbSent = 0; + + # Do a timed send. + msStart = base.timestampMilli(); + while True: + cMsElapsed = base.timestampMilli() - msStart; + if cMsElapsed > cMsTimeout: + reporter.error('TranportTcp.sendBytes: %s bytes timed out (1)' % (len(abBuf))); + break; + + # wait. + try: + ttRc = select.select([], [self.oSocket], [self.oSocket], (cMsTimeout - cMsElapsed) / 1000.0); + if ttRc[2] and not ttRc[1]: + reporter.error('TranportTcp.sendBytes: select returned with exception'); + break; + if not ttRc[1]: + reporter.error('TranportTcp.sendBytes: %s bytes timed out (2)' % (len(abBuf))); + break; + except: + reporter.errorXcpt('TranportTcp.sendBytes: select failed'); + break; + + # Try send more. + try: + cbSent += self.oSocket.send(abBuf[cbSent:]); + if cbSent == len(abBuf): + return True; + except Exception as oXcpt: + if not self.__isWouldBlockXcpt(oXcpt): + reporter.errorXcpt('TranportTcp.sendBytes: %s bytes' % (len(abBuf))); + break; + + return False; + + def __returnReadAheadBytes(self, cb): + """ Internal worker for recvBytes. """ + assert(len(self.abReadAhead) >= cb); + abRet = self.abReadAhead[:cb]; + self.abReadAhead = self.abReadAhead[cb:]; + return abRet; + + def recvBytes(self, cb, cMsTimeout, fNoDataOk): + if self.oSocket is None: + reporter.error('TransportTcp.recvBytes(%s,%s): No connection.' % (cb, cMsTimeout)); + return None; + + # Try read in some more data without bothering with timeout handling first. + if len(self.abReadAhead) < cb: + try: + abBuf = self.oSocket.recv(cb - len(self.abReadAhead)); + if abBuf: + self.abReadAhead.extend(array.array('B', abBuf)); + except Exception as oXcpt: + if not self.__isWouldBlockXcpt(oXcpt): + reporter.errorXcpt('TranportTcp.recvBytes: 0/%s bytes' % (cb,)); + return None; + + if len(self.abReadAhead) >= cb: + return self.__returnReadAheadBytes(cb); + + # Timeout loop. + msStart = base.timestampMilli(); + while True: + cMsElapsed = base.timestampMilli() - msStart; + if cMsElapsed > cMsTimeout: + if not fNoDataOk or self.abReadAhead: + reporter.error('TranportTcp.recvBytes: %s/%s bytes timed out (1)' % (len(self.abReadAhead), cb)); + break; + + # Wait. + try: + ttRc = select.select([self.oSocket], [], [self.oSocket], (cMsTimeout - cMsElapsed) / 1000.0); + if ttRc[2] and not ttRc[0]: + reporter.error('TranportTcp.recvBytes: select returned with exception'); + break; + if not ttRc[0]: + if not fNoDataOk or self.abReadAhead: + reporter.error('TranportTcp.recvBytes: %s/%s bytes timed out (2) fNoDataOk=%s' + % (len(self.abReadAhead), cb, fNoDataOk)); + break; + except: + reporter.errorXcpt('TranportTcp.recvBytes: select failed'); + break; + + # Try read more. + try: + abBuf = self.oSocket.recv(cb - len(self.abReadAhead)); + if not abBuf: + reporter.error('TranportTcp.recvBytes: %s/%s bytes (%s) - connection has been shut down' + % (len(self.abReadAhead), cb, fNoDataOk)); + self.disconnect(); + return None; + + self.abReadAhead.extend(array.array('B', abBuf)); + + except Exception as oXcpt: + reporter.log('recv => exception %s' % (oXcpt,)); + if not self.__isWouldBlockXcpt(oXcpt): + if not fNoDataOk or not self.__isConnectionReset(oXcpt) or self.abReadAhead: + reporter.errorXcpt('TranportTcp.recvBytes: %s/%s bytes (%s)' % (len(self.abReadAhead), cb, fNoDataOk)); + break; + + # Done? + if len(self.abReadAhead) >= cb: + return self.__returnReadAheadBytes(cb); + + #reporter.log('recv => None len(self.abReadAhead) -> %d' % (len(self.abReadAhead), )); + return None; + + def isConnectionOk(self): + if self.oSocket is None: + return False; + try: + ttRc = select.select([], [], [self.oSocket], 0.0); + if ttRc[2]: + return False; + + self.oSocket.send(array.array('B')); # send zero bytes. + except: + return False; + return True; + + def isRecvPending(self, cMsTimeout = 0): + try: + ttRc = select.select([self.oSocket], [], [], cMsTimeout / 1000.0); + if not ttRc[0]: + return False; + except: + pass; + return True; + + +def openTcpSession(cMsTimeout, sHostname, uPort = None, fReversedSetup = False, cMsIdleFudge = 0, fnProcessEvents = None): + """ + Opens a connection to a Test Execution Service via TCP, given its name. + + The optional fnProcessEvents callback should be set to vbox.processPendingEvents + or similar. + """ + reporter.log2('openTcpSession(%s, %s, %s, %s, %s)' % + (cMsTimeout, sHostname, uPort, fReversedSetup, cMsIdleFudge)); + try: + oTransport = TransportTcp(sHostname, uPort, fReversedSetup); + oSession = Session(oTransport, cMsTimeout, cMsIdleFudge, fnProcessEvents = fnProcessEvents); + except: + reporter.errorXcpt(None, 15); + return None; + return oSession; + + +def tryOpenTcpSession(cMsTimeout, sHostname, uPort = None, fReversedSetup = False, cMsIdleFudge = 0, fnProcessEvents = None): + """ + Tries to open a connection to a Test Execution Service via TCP, given its name. + + This differs from openTcpSession in that it won't log a connection failure + as an error. + """ + try: + oTransport = TransportTcp(sHostname, uPort, fReversedSetup); + oSession = Session(oTransport, cMsTimeout, cMsIdleFudge, fTryConnect = True, fnProcessEvents = fnProcessEvents); + except: + reporter.errorXcpt(None, 15); + return None; + return oSession; diff --git a/src/VBox/ValidationKit/testdriver/vbox.py b/src/VBox/ValidationKit/testdriver/vbox.py new file mode 100755 index 00000000..89bb7ff8 --- /dev/null +++ b/src/VBox/ValidationKit/testdriver/vbox.py @@ -0,0 +1,4569 @@ +# -*- coding: utf-8 -*- +# $Id: vbox.py $ +# pylint: disable=too-many-lines + +""" +VirtualBox Specific base testdriver. +""" + +__copyright__ = \ +""" +Copyright (C) 2010-2022 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 154728 $" + +# pylint: disable=unnecessary-semicolon + +# Standard Python imports. +import datetime +import os +import platform +import re; +import sys +import threading +import time +import traceback + +# Figure out where the validation kit lives and make sure it's in the path. +try: __file__ +except: __file__ = sys.argv[0]; +g_ksValidationKitDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))); +if g_ksValidationKitDir not in sys.path: + sys.path.append(g_ksValidationKitDir); + +# Validation Kit imports. +from common import utils; +from testdriver import base; +from testdriver import btresolver; +from testdriver import reporter; +from testdriver import vboxcon; +from testdriver import vboxtestvms; + +# Python 3 hacks: +if sys.version_info[0] >= 3: + xrange = range; # pylint: disable=redefined-builtin,invalid-name + long = int; # pylint: disable=redefined-builtin,invalid-name + +# +# Exception and Error Unification Hacks. +# Note! This is pretty gross stuff. Be warned! +# TODO: Find better ways of doing these things, preferrably in vboxapi. +# + +ComException = None; # pylint: disable=invalid-name +__fnComExceptionGetAttr__ = None; # pylint: disable=invalid-name + +def __MyDefaultGetAttr(oSelf, sName): + """ __getattribute__/__getattr__ default fake.""" + try: + oAttr = oSelf.__dict__[sName]; + except: + oAttr = dir(oSelf)[sName]; + return oAttr; + +def __MyComExceptionGetAttr(oSelf, sName): + """ ComException.__getattr__ wrapper - both XPCOM and COM. """ + try: + oAttr = __fnComExceptionGetAttr__(oSelf, sName); + except AttributeError: + if platform.system() == 'Windows': + if sName == 'errno': + oAttr = __fnComExceptionGetAttr__(oSelf, 'hresult'); + elif sName == 'msg': + oAttr = __fnComExceptionGetAttr__(oSelf, 'strerror'); + else: + raise; + else: + if sName == 'hresult': + oAttr = __fnComExceptionGetAttr__(oSelf, 'errno'); + elif sName == 'strerror': + oAttr = __fnComExceptionGetAttr__(oSelf, 'msg'); + elif sName == 'excepinfo': + oAttr = None; + elif sName == 'argerror': + oAttr = None; + else: + raise; + #print '__MyComExceptionGetAttr(,%s) -> "%s"' % (sName, oAttr); + return oAttr; + +def __deployExceptionHacks__(oNativeComExceptionClass): + """ + Deploys the exception and error hacks that helps unifying COM and XPCOM + exceptions and errors. + """ + global ComException # pylint: disable=invalid-name + global __fnComExceptionGetAttr__ # pylint: disable=invalid-name + + # Hook up our attribute getter for the exception class (ASSUMES new-style). + if __fnComExceptionGetAttr__ is None: + try: + __fnComExceptionGetAttr__ = getattr(oNativeComExceptionClass, '__getattr__'); + except: + try: + __fnComExceptionGetAttr__ = getattr(oNativeComExceptionClass, '__getattribute__'); + except: + __fnComExceptionGetAttr__ = __MyDefaultGetAttr; + setattr(oNativeComExceptionClass, '__getattr__', __MyComExceptionGetAttr) + + # Make the modified classes accessible (are there better ways to do this?) + ComException = oNativeComExceptionClass + return None; + + + +# +# Utility functions. +# + +def isIpAddrValid(sIpAddr): + """ + Checks if a IPv4 address looks valid. This will return false for + localhost and similar. + Returns True / False. + """ + if sIpAddr is None: return False; + if len(sIpAddr.split('.')) != 4: return False; + if sIpAddr.endswith('.0'): return False; + if sIpAddr.endswith('.255'): return False; + if sIpAddr.startswith('127.'): return False; + if sIpAddr.startswith('169.254.'): return False; + if sIpAddr.startswith('192.0.2.'): return False; + if sIpAddr.startswith('224.0.0.'): return False; + return True; + +def stringifyErrorInfo(oErrInfo): + """ + Stringifies the error information in a IVirtualBoxErrorInfo object. + + Returns string with error info. + """ + try: + rc = oErrInfo.resultCode; + sText = oErrInfo.text; + sIid = oErrInfo.interfaceID; + sComponent = oErrInfo.component; + except: + sRet = 'bad error object (%s)?' % (oErrInfo,); + traceback.print_exc(); + else: + sRet = 'rc=%s text="%s" IID=%s component=%s' % (ComError.toString(rc), sText, sIid, sComponent); + return sRet; + +def reportError(oErr, sText): + """ + Report a VirtualBox error on oErr. oErr can be IVirtualBoxErrorInfo + or IProgress. Anything else is ignored. + + Returns the same a reporter.error(). + """ + try: + oErrObj = oErr.errorInfo; # IProgress. + except: + oErrObj = oErr; + reporter.error(sText); + return reporter.error(stringifyErrorInfo(oErrObj)); + +def formatComOrXpComException(oType, oXcpt): + """ + Callback installed with the reporter to better format COM exceptions. + Similar to format_exception_only, only it returns None if not interested. + """ + _ = oType; + oVBoxMgr = vboxcon.goHackModuleClass.oVBoxMgr; + if oVBoxMgr is None: + return None; + if not oVBoxMgr.xcptIsOurXcptKind(oXcpt): # pylint: disable=not-callable + return None; + + if platform.system() == 'Windows': + hrc = oXcpt.hresult; + if hrc == ComError.DISP_E_EXCEPTION and oXcpt.excepinfo is not None and len(oXcpt.excepinfo) > 5: + hrc = oXcpt.excepinfo[5]; + sWhere = oXcpt.excepinfo[1]; + sMsg = oXcpt.excepinfo[2]; + else: + sWhere = None; + sMsg = oXcpt.strerror; + else: + hrc = oXcpt.errno; + sWhere = None; + sMsg = oXcpt.msg; + + sHrc = oVBoxMgr.xcptToString(hrc); # pylint: disable=not-callable + if sHrc.find('(') < 0: + sHrc = '%s (%#x)' % (sHrc, hrc & 0xffffffff,); + + asRet = ['COM-Xcpt: %s' % (sHrc,)]; + if sMsg and sWhere: + asRet.append('--------- %s: %s' % (sWhere, sMsg,)); + elif sMsg: + asRet.append('--------- %s' % (sMsg,)); + return asRet; + #if sMsg and sWhere: + # return ['COM-Xcpt: %s - %s: %s' % (sHrc, sWhere, sMsg,)]; + #if sMsg: + # return ['COM-Xcpt: %s - %s' % (sHrc, sMsg,)]; + #return ['COM-Xcpt: %s' % (sHrc,)]; + +# +# Classes +# + +class ComError(object): + """ + Unified COM and XPCOM status code repository. + This works more like a module than a class since it's replacing a module. + """ + + # The VBOX_E_XXX bits: + __VBOX_E_BASE = -2135228416; + VBOX_E_OBJECT_NOT_FOUND = __VBOX_E_BASE + 1; + VBOX_E_INVALID_VM_STATE = __VBOX_E_BASE + 2; + VBOX_E_VM_ERROR = __VBOX_E_BASE + 3; + VBOX_E_FILE_ERROR = __VBOX_E_BASE + 4; + VBOX_E_IPRT_ERROR = __VBOX_E_BASE + 5; + VBOX_E_PDM_ERROR = __VBOX_E_BASE + 6; + VBOX_E_INVALID_OBJECT_STATE = __VBOX_E_BASE + 7; + VBOX_E_HOST_ERROR = __VBOX_E_BASE + 8; + VBOX_E_NOT_SUPPORTED = __VBOX_E_BASE + 9; + VBOX_E_XML_ERROR = __VBOX_E_BASE + 10; + VBOX_E_INVALID_SESSION_STATE = __VBOX_E_BASE + 11; + VBOX_E_OBJECT_IN_USE = __VBOX_E_BASE + 12; + VBOX_E_DONT_CALL_AGAIN = __VBOX_E_BASE + 13; + + # Reverse lookup table. + dDecimalToConst = {}; # pylint: disable=invalid-name + + def __init__(self): + raise base.GenError('No instances, please'); + + @staticmethod + def copyErrors(oNativeComErrorClass): + """ + Copy all error codes from oNativeComErrorClass to this class and + install compatability mappings. + """ + + # First, add the VBOX_E_XXX constants to dDecimalToConst. + for sAttr in dir(ComError): + if sAttr.startswith('VBOX_E'): + oAttr = getattr(ComError, sAttr); + ComError.dDecimalToConst[oAttr] = sAttr; + + # Copy all error codes from oNativeComErrorClass to this class. + for sAttr in dir(oNativeComErrorClass): + if sAttr[0].isupper(): + oAttr = getattr(oNativeComErrorClass, sAttr); + setattr(ComError, sAttr, oAttr); + if isinstance(oAttr, int): + ComError.dDecimalToConst[oAttr] = sAttr; + + # Install mappings to the other platform. + if platform.system() == 'Windows': + ComError.NS_OK = ComError.S_OK; + ComError.NS_ERROR_FAILURE = ComError.E_FAIL; + ComError.NS_ERROR_ABORT = ComError.E_ABORT; + ComError.NS_ERROR_NULL_POINTER = ComError.E_POINTER; + ComError.NS_ERROR_NO_INTERFACE = ComError.E_NOINTERFACE; + ComError.NS_ERROR_INVALID_ARG = ComError.E_INVALIDARG; + ComError.NS_ERROR_OUT_OF_MEMORY = ComError.E_OUTOFMEMORY; + ComError.NS_ERROR_NOT_IMPLEMENTED = ComError.E_NOTIMPL; + ComError.NS_ERROR_UNEXPECTED = ComError.E_UNEXPECTED; + else: + ComError.E_ACCESSDENIED = -2147024891; # see VBox/com/defs.h + ComError.S_OK = ComError.NS_OK; + ComError.E_FAIL = ComError.NS_ERROR_FAILURE; + ComError.E_ABORT = ComError.NS_ERROR_ABORT; + ComError.E_POINTER = ComError.NS_ERROR_NULL_POINTER; + ComError.E_NOINTERFACE = ComError.NS_ERROR_NO_INTERFACE; + ComError.E_INVALIDARG = ComError.NS_ERROR_INVALID_ARG; + ComError.E_OUTOFMEMORY = ComError.NS_ERROR_OUT_OF_MEMORY; + ComError.E_NOTIMPL = ComError.NS_ERROR_NOT_IMPLEMENTED; + ComError.E_UNEXPECTED = ComError.NS_ERROR_UNEXPECTED; + ComError.DISP_E_EXCEPTION = -2147352567; # For COM compatability only. + return True; + + @staticmethod + def getXcptResult(oXcpt): + """ + Gets the result code for an exception. + Returns COM status code (or E_UNEXPECTED). + """ + if platform.system() == 'Windows': + # The DISP_E_EXCEPTION + excptinfo fun needs checking up, only + # empirical info on it so far. + try: + hrXcpt = oXcpt.hresult; + except AttributeError: + hrXcpt = ComError.E_UNEXPECTED; + if hrXcpt == ComError.DISP_E_EXCEPTION and oXcpt.excepinfo is not None: + hrXcpt = oXcpt.excepinfo[5]; + else: + try: + hrXcpt = oXcpt.errno; + except AttributeError: + hrXcpt = ComError.E_UNEXPECTED; + return hrXcpt; + + @staticmethod + def equal(oXcpt, hr): + """ + Checks if the ComException e is not equal to the COM status code hr. + This takes DISP_E_EXCEPTION & excepinfo into account. + + This method can be used with any Exception derivate, however it will + only return True for classes similar to the two ComException variants. + """ + if platform.system() == 'Windows': + # The DISP_E_EXCEPTION + excptinfo fun needs checking up, only + # empirical info on it so far. + try: + hrXcpt = oXcpt.hresult; + except AttributeError: + return False; + if hrXcpt == ComError.DISP_E_EXCEPTION and oXcpt.excepinfo is not None: + hrXcpt = oXcpt.excepinfo[5]; + else: + try: + hrXcpt = oXcpt.errno; + except AttributeError: + return False; + return hrXcpt == hr; + + @staticmethod + def notEqual(oXcpt, hr): + """ + Checks if the ComException e is not equal to the COM status code hr. + See equal() for more details. + """ + return not ComError.equal(oXcpt, hr) + + @staticmethod + def toString(hr): + """ + Converts the specified COM status code to a string. + """ + try: + sStr = ComError.dDecimalToConst[int(hr)]; + except KeyError: + hrLong = long(hr); + sStr = '%#x (%d)' % (hrLong, hrLong); + return sStr; + + +class Build(object): # pylint: disable=too-few-public-methods + """ + A VirtualBox build. + + Note! After dropping the installation of VBox from this code and instead + realizing that with the vboxinstall.py wrapper driver, this class is + of much less importance and contains unnecessary bits and pieces. + """ + + def __init__(self, oDriver, strInstallPath): + """ + Construct a build object from a build file name and/or install path. + """ + # Initialize all members first. + self.oDriver = oDriver; + self.sInstallPath = strInstallPath; + self.sSdkPath = None; + self.sSrcRoot = None; + self.sKind = None; + self.sDesignation = None; + self.sType = None; + self.sOs = None; + self.sArch = None; + self.sGuestAdditionsIso = None; + + # Figure out the values as best we can. + if strInstallPath is None: + # + # Both parameters are None, which means we're falling back on a + # build in the development tree. + # + self.sKind = "development"; + + if self.sType is None: + self.sType = os.environ.get("KBUILD_TYPE", "release"); + if self.sOs is None: + self.sOs = os.environ.get("KBUILD_TARGET", oDriver.sHost); + if self.sArch is None: + self.sArch = os.environ.get("KBUILD_TARGET_ARCH", oDriver.sHostArch); + + sOut = os.path.join('out', self.sOs + '.' + self.sArch, self.sType); + sSearch = os.environ.get('VBOX_TD_DEV_TREE', os.path.dirname(__file__)); # Env.var. for older trees or testboxscript. + sCandidat = None; + for i in range(0, 10): # pylint: disable=unused-variable + sBldDir = os.path.join(sSearch, sOut); + if os.path.isdir(sBldDir): + sCandidat = os.path.join(sBldDir, 'bin', 'VBoxSVC' + base.exeSuff()); + if os.path.isfile(sCandidat): + self.sSdkPath = os.path.join(sBldDir, 'bin/sdk'); + break; + sCandidat = os.path.join(sBldDir, 'dist/VirtualBox.app/Contents/MacOS/VBoxSVC'); + if os.path.isfile(sCandidat): + self.sSdkPath = os.path.join(sBldDir, 'dist/sdk'); + break; + sSearch = os.path.abspath(os.path.join(sSearch, '..')); + if sCandidat is None or not os.path.isfile(sCandidat): + raise base.GenError(); + self.sInstallPath = os.path.abspath(os.path.dirname(sCandidat)); + self.sSrcRoot = os.path.abspath(sSearch); + + self.sDesignation = os.environ.get('TEST_BUILD_DESIGNATION', None); + if self.sDesignation is None: + try: + oFile = utils.openNoInherit(os.path.join(self.sSrcRoot, sOut, 'revision.kmk'), 'r'); + except: + pass; + else: + s = oFile.readline(); + oFile.close(); + oMatch = re.search("VBOX_SVN_REV=(\\d+)", s); + if oMatch is not None: + self.sDesignation = oMatch.group(1); + + if self.sDesignation is None: + self.sDesignation = 'XXXXX' + else: + # + # We've been pointed to an existing installation, this could be + # in the out dir of a svn checkout, untarred VBoxAll or a real + # installation directory. + # + self.sKind = "preinstalled"; + self.sType = "release"; + self.sOs = oDriver.sHost; + self.sArch = oDriver.sHostArch; + self.sInstallPath = os.path.abspath(strInstallPath); + self.sSdkPath = os.path.join(self.sInstallPath, 'sdk'); + self.sSrcRoot = None; + self.sDesignation = os.environ.get('TEST_BUILD_DESIGNATION', 'XXXXX'); + ## @todo Much more work is required here. + + # Try Determine the build type. + sVBoxManage = os.path.join(self.sInstallPath, 'VBoxManage' + base.exeSuff()); + if os.path.isfile(sVBoxManage): + try: + (iExit, sStdOut, _) = utils.processOutputUnchecked([sVBoxManage, '--dump-build-type']); + sStdOut = sStdOut.strip(); + if iExit == 0 and sStdOut in ('release', 'debug', 'strict', 'dbgopt', 'asan'): + self.sType = sStdOut; + reporter.log('Build: Detected build type: %s' % (self.sType)); + else: + reporter.log('Build: --dump-build-type -> iExit=%u sStdOut=%s' % (iExit, sStdOut,)); + except: + reporter.logXcpt('Build: Running "%s --dump-build-type" failed!' % (sVBoxManage,)); + else: + reporter.log3('Build: sVBoxManage=%s not found' % (sVBoxManage,)); + + # Do some checks. + sVMMR0 = os.path.join(self.sInstallPath, 'VMMR0.r0'); + if not os.path.isfile(sVMMR0) and utils.getHostOs() == 'solaris': # solaris is special. + sVMMR0 = os.path.join(self.sInstallPath, 'amd64' if utils.getHostArch() == 'amd64' else 'i386', 'VMMR0.r0'); + if not os.path.isfile(sVMMR0): + raise base.GenError('%s is missing' % (sVMMR0,)); + + # Guest additions location is different on windows for some _stupid_ reason. + if self.sOs == 'win' and self.sKind != 'development': + self.sGuestAdditionsIso = '%s/VBoxGuestAdditions.iso' % (self.sInstallPath,); + elif self.sOs == 'darwin': + self.sGuestAdditionsIso = '%s/VBoxGuestAdditions.iso' % (self.sInstallPath,); + elif self.sOs == 'solaris': + self.sGuestAdditionsIso = '%s/VBoxGuestAdditions.iso' % (self.sInstallPath,); + else: + self.sGuestAdditionsIso = '%s/additions/VBoxGuestAdditions.iso' % (self.sInstallPath,); + + # __init__ end; + + def isDevBuild(self): + """ Returns True if it's development build (kind), otherwise False. """ + return self.sKind == 'development'; + + +class EventHandlerBase(object): + """ + Base class for both Console and VirtualBox event handlers. + """ + + def __init__(self, dArgs, fpApiVer, sName = None): + self.oVBoxMgr = dArgs['oVBoxMgr']; + self.oEventSrc = dArgs['oEventSrc']; # Console/VirtualBox for < 3.3 + self.oListener = dArgs['oListener']; + self.fPassive = self.oListener is not None; + self.sName = sName + self.fShutdown = False; + self.oThread = None; + self.fpApiVer = fpApiVer; + self.dEventNo2Name = {}; + for sKey, iValue in self.oVBoxMgr.constants.all_values('VBoxEventType').items(): + self.dEventNo2Name[iValue] = sKey; + + def threadForPassiveMode(self): + """ + The thread procedure for the event processing thread. + """ + assert self.fPassive is not None; + while not self.fShutdown: + try: + oEvt = self.oEventSrc.getEvent(self.oListener, 500); + except: + if not self.oVBoxMgr.xcptIsDeadInterface(): reporter.logXcpt(); + else: reporter.log('threadForPassiveMode/%s: interface croaked (ignored)' % (self.sName,)); + break; + if oEvt: + self.handleEvent(oEvt); + if not self.fShutdown: + try: + self.oEventSrc.eventProcessed(self.oListener, oEvt); + except: + reporter.logXcpt(); + break; + self.unregister(fWaitForThread = False); + return None; + + def startThreadForPassiveMode(self): + """ + Called when working in passive mode. + """ + self.oThread = threading.Thread(target = self.threadForPassiveMode, \ + args=(), name=('PAS-%s' % (self.sName,))); + self.oThread.setDaemon(True); # pylint: disable=deprecated-method + self.oThread.start(); + return None; + + def unregister(self, fWaitForThread = True): + """ + Unregister the event handler. + """ + fRc = False; + if not self.fShutdown: + self.fShutdown = True; + + if self.oEventSrc is not None: + if self.fpApiVer < 3.3: + try: + self.oEventSrc.unregisterCallback(self.oListener); + fRc = True; + except: + reporter.errorXcpt('unregisterCallback failed on %s' % (self.oListener,)); + else: + try: + self.oEventSrc.unregisterListener(self.oListener); + fRc = True; + except: + if self.oVBoxMgr.xcptIsDeadInterface(): + reporter.log('unregisterListener failed on %s because of dead interface (%s)' + % (self.oListener, self.oVBoxMgr.xcptToString(),)); + else: + reporter.errorXcpt('unregisterListener failed on %s' % (self.oListener,)); + + if self.oThread is not None \ + and self.oThread != threading.current_thread(): + self.oThread.join(); + self.oThread = None; + + _ = fWaitForThread; + return fRc; + + def handleEvent(self, oEvt): + """ + Compatibility wrapper that child classes implement. + """ + _ = oEvt; + return None; + + @staticmethod + def registerDerivedEventHandler(oVBoxMgr, fpApiVer, oSubClass, dArgsCopy, # pylint: disable=too-many-arguments + oSrcParent, sSrcParentNm, sICallbackNm, + fMustSucceed = True, sLogSuffix = '', aenmEvents = None): + """ + Registers the callback / event listener. + """ + dArgsCopy['oVBoxMgr'] = oVBoxMgr; + dArgsCopy['oListener'] = None; + if fpApiVer < 3.3: + dArgsCopy['oEventSrc'] = oSrcParent; + try: + oRet = oVBoxMgr.createCallback(sICallbackNm, oSubClass, dArgsCopy); + except: + reporter.errorXcpt('%s::registerCallback(%s) failed%s' % (sSrcParentNm, oRet, sLogSuffix)); + else: + try: + oSrcParent.registerCallback(oRet); + return oRet; + except Exception as oXcpt: + if fMustSucceed or ComError.notEqual(oXcpt, ComError.E_UNEXPECTED): + reporter.errorXcpt('%s::registerCallback(%s)%s' % (sSrcParentNm, oRet, sLogSuffix)); + else: + # + # Scalable event handling introduced in VBox 4.0. + # + fPassive = sys.platform == 'win32'; # or webservices. + + if not aenmEvents: + aenmEvents = (vboxcon.VBoxEventType_Any,); + + try: + oEventSrc = oSrcParent.eventSource; + dArgsCopy['oEventSrc'] = oEventSrc; + if not fPassive: + oListener = oRet = oVBoxMgr.createListener(oSubClass, dArgsCopy); + else: + oListener = oEventSrc.createListener(); + dArgsCopy['oListener'] = oListener; + oRet = oSubClass(dArgsCopy); + except: + reporter.errorXcpt('%s::eventSource.createListener(%s) failed%s' % (sSrcParentNm, oListener, sLogSuffix)); + else: + try: + oEventSrc.registerListener(oListener, aenmEvents, not fPassive); + except Exception as oXcpt: + if fMustSucceed or ComError.notEqual(oXcpt, ComError.E_UNEXPECTED): + reporter.errorXcpt('%s::eventSource.registerListener(%s) failed%s' + % (sSrcParentNm, oListener, sLogSuffix)); + else: + if not fPassive: + if sys.platform == 'win32': + from win32com.server.util import unwrap # pylint: disable=import-error + oRet = unwrap(oRet); + oRet.oListener = oListener; + else: + oRet.startThreadForPassiveMode(); + return oRet; + return None; + + + + +class ConsoleEventHandlerBase(EventHandlerBase): + """ + Base class for handling IConsole events. + + The class has IConsoleCallback (<=3.2) compatible callback methods which + the user can override as needed. + + Note! This class must not inherit from object or we'll get type errors in VBoxPython. + """ + def __init__(self, dArgs, sName = None): + self.oSession = dArgs['oSession']; + self.oConsole = dArgs['oConsole']; + if sName is None: + sName = self.oSession.sName; + EventHandlerBase.__init__(self, dArgs, self.oSession.fpApiVer, sName); + + + # pylint: disable=missing-docstring,too-many-arguments,unused-argument + def onMousePointerShapeChange(self, fVisible, fAlpha, xHot, yHot, cx, cy, abShape): + reporter.log2('onMousePointerShapeChange/%s' % (self.sName)); + def onMouseCapabilityChange(self, fSupportsAbsolute, *aArgs): # Extra argument was added in 3.2. + reporter.log2('onMouseCapabilityChange/%s' % (self.sName)); + def onKeyboardLedsChange(self, fNumLock, fCapsLock, fScrollLock): + reporter.log2('onKeyboardLedsChange/%s' % (self.sName)); + def onStateChange(self, eState): + reporter.log2('onStateChange/%s' % (self.sName)); + def onAdditionsStateChange(self): + reporter.log2('onAdditionsStateChange/%s' % (self.sName)); + def onNetworkAdapterChange(self, oNic): + reporter.log2('onNetworkAdapterChange/%s' % (self.sName)); + def onSerialPortChange(self, oPort): + reporter.log2('onSerialPortChange/%s' % (self.sName)); + def onParallelPortChange(self, oPort): + reporter.log2('onParallelPortChange/%s' % (self.sName)); + def onStorageControllerChange(self): + reporter.log2('onStorageControllerChange/%s' % (self.sName)); + def onMediumChange(self, attachment): + reporter.log2('onMediumChange/%s' % (self.sName)); + def onCPUChange(self, iCpu, fAdd): + reporter.log2('onCPUChange/%s' % (self.sName)); + def onVRDPServerChange(self): + reporter.log2('onVRDPServerChange/%s' % (self.sName)); + def onRemoteDisplayInfoChange(self): + reporter.log2('onRemoteDisplayInfoChange/%s' % (self.sName)); + def onUSBControllerChange(self): + reporter.log2('onUSBControllerChange/%s' % (self.sName)); + def onUSBDeviceStateChange(self, oDevice, fAttached, oError): + reporter.log2('onUSBDeviceStateChange/%s' % (self.sName)); + def onSharedFolderChange(self, fGlobal): + reporter.log2('onSharedFolderChange/%s' % (self.sName)); + def onRuntimeError(self, fFatal, sErrId, sMessage): + reporter.log2('onRuntimeError/%s' % (self.sName)); + def onCanShowWindow(self): + reporter.log2('onCanShowWindow/%s' % (self.sName)); + return True + def onShowWindow(self): + reporter.log2('onShowWindow/%s' % (self.sName)); + return None; + # pylint: enable=missing-docstring,too-many-arguments,unused-argument + + def handleEvent(self, oEvt): + """ + Compatibility wrapper. + """ + try: + oEvtBase = self.oVBoxMgr.queryInterface(oEvt, 'IEvent'); + eType = oEvtBase.type; + except: + reporter.logXcpt(); + return None; + if eType == vboxcon.VBoxEventType_OnRuntimeError: + try: + oEvtIt = self.oVBoxMgr.queryInterface(oEvtBase, 'IRuntimeErrorEvent'); + return self.onRuntimeError(oEvtIt.fatal, oEvtIt.id, oEvtIt.message) + except: + reporter.logXcpt(); + ## @todo implement the other events. + try: + if eType not in (vboxcon.VBoxEventType_OnMousePointerShapeChanged, + vboxcon.VBoxEventType_OnCursorPositionChanged): + if eType in self.dEventNo2Name: + reporter.log2('%s(%s)/%s' % (self.dEventNo2Name[eType], str(eType), self.sName)); + else: + reporter.log2('%s/%s' % (str(eType), self.sName)); + except AttributeError: # Handle older VBox versions which don't have a specific event. + pass; + return None; + + +class VirtualBoxEventHandlerBase(EventHandlerBase): + """ + Base class for handling IVirtualBox events. + + The class has IConsoleCallback (<=3.2) compatible callback methods which + the user can override as needed. + + Note! This class must not inherit from object or we'll get type errors in VBoxPython. + """ + def __init__(self, dArgs, sName = "emanon"): + self.oVBoxMgr = dArgs['oVBoxMgr']; + self.oVBox = dArgs['oVBox']; + EventHandlerBase.__init__(self, dArgs, self.oVBox.fpApiVer, sName); + + # pylint: disable=missing-docstring,unused-argument + def onMachineStateChange(self, sMachineId, eState): + pass; + def onMachineDataChange(self, sMachineId): + pass; + def onExtraDataCanChange(self, sMachineId, sKey, sValue): + # The COM bridge does tuples differently. Not very funny if you ask me... ;-) + if self.oVBoxMgr.type == 'MSCOM': + return '', 0, True; + return True, '' + def onExtraDataChange(self, sMachineId, sKey, sValue): + pass; + def onMediumRegistered(self, sMediumId, eMediumType, fRegistered): + pass; + def onMachineRegistered(self, sMachineId, fRegistered): + pass; + def onSessionStateChange(self, sMachineId, eState): + pass; + def onSnapshotTaken(self, sMachineId, sSnapshotId): + pass; + def onSnapshotDiscarded(self, sMachineId, sSnapshotId): + pass; + def onSnapshotChange(self, sMachineId, sSnapshotId): + pass; + def onGuestPropertyChange(self, sMachineId, sName, sValue, sFlags, fWasDeleted): + pass; + # pylint: enable=missing-docstring,unused-argument + + def handleEvent(self, oEvt): + """ + Compatibility wrapper. + """ + try: + oEvtBase = self.oVBoxMgr.queryInterface(oEvt, 'IEvent'); + eType = oEvtBase.type; + except: + reporter.logXcpt(); + return None; + if eType == vboxcon.VBoxEventType_OnMachineStateChanged: + try: + oEvtIt = self.oVBoxMgr.queryInterface(oEvtBase, 'IMachineStateChangedEvent'); + return self.onMachineStateChange(oEvtIt.machineId, oEvtIt.state) + except: + reporter.logXcpt(); + elif eType == vboxcon.VBoxEventType_OnGuestPropertyChanged: + try: + oEvtIt = self.oVBoxMgr.queryInterface(oEvtBase, 'IGuestPropertyChangedEvent'); + if hasattr(oEvtIt, 'fWasDeleted'): # Since 7.0 we have a dedicated flag + fWasDeleted = oEvtIt.fWasDeleted; + else: + fWasDeleted = False; # Don't indicate deletion here -- there can be empty guest properties. + return self.onGuestPropertyChange(oEvtIt.machineId, oEvtIt.name, oEvtIt.value, oEvtIt.flags, fWasDeleted); + except: + reporter.logXcpt(); + ## @todo implement the other events. + if eType in self.dEventNo2Name: + reporter.log2('%s(%s)/%s' % (self.dEventNo2Name[eType], str(eType), self.sName)); + else: + reporter.log2('%s/%s' % (str(eType), self.sName)); + return None; + + +class SessionConsoleEventHandler(ConsoleEventHandlerBase): + """ + For catching machine state changes and waking up the task machinery at that point. + """ + def __init__(self, dArgs): + ConsoleEventHandlerBase.__init__(self, dArgs); + + def onMachineStateChange(self, sMachineId, eState): # pylint: disable=unused-argument + """ Just interrupt the wait loop here so it can check again. """ + _ = sMachineId; _ = eState; + self.oVBoxMgr.interruptWaitEvents(); + + def onRuntimeError(self, fFatal, sErrId, sMessage): + reporter.log('onRuntimeError/%s: fFatal=%d sErrId=%s sMessage=%s' % (self.sName, fFatal, sErrId, sMessage)); + oSession = self.oSession; + if oSession is not None: # paranoia + if sErrId == 'HostMemoryLow': + oSession.signalHostMemoryLow(); + if sys.platform == 'win32': + from testdriver import winbase; + winbase.logMemoryStats(); + oSession.signalTask(); + self.oVBoxMgr.interruptWaitEvents(); + + + +class TestDriver(base.TestDriver): # pylint: disable=too-many-instance-attributes + """ + This is the VirtualBox test driver. + """ + + def __init__(self): + base.TestDriver.__init__(self); + self.fImportedVBoxApi = False; + self.fpApiVer = 3.2; + self.uRevision = 0; + self.uApiRevision = 0; + self.oBuild = None; + self.oVBoxMgr = None; + self.oVBox = None; + self.aoRemoteSessions = []; + self.aoVMs = []; ## @todo not sure if this list will be of any use. + self.oTestVmManager = vboxtestvms.TestVmManager(self.sResourcePath); + self.oTestVmSet = vboxtestvms.TestVmSet(); + self.sSessionTypeDef = 'headless'; + self.sSessionType = self.sSessionTypeDef; + self.fEnableVrdp = True; + self.uVrdpBasePortDef = 6000; + self.uVrdpBasePort = self.uVrdpBasePortDef; + self.sDefBridgedNic = None; + self.fUseDefaultSvc = False; + self.sLogSelfGroups = ''; + self.sLogSelfFlags = 'time'; + self.sLogSelfDest = ''; + self.sLogSessionGroups = ''; + self.sLogSessionFlags = 'time'; + self.sLogSessionDest = ''; + self.sLogSvcGroups = ''; + self.sLogSvcFlags = 'time'; + self.sLogSvcDest = ''; + self.sSelfLogFile = None; + self.sSessionLogFile = None; + self.sVBoxSvcLogFile = None; + self.oVBoxSvcProcess = None; + self.sVBoxSvcPidFile = None; + self.fVBoxSvcInDebugger = False; + self.fVBoxSvcWaitForDebugger = False; + self.sVBoxValidationKit = None; + self.sVBoxValidationKitIso = None; + self.sVBoxBootSectors = None; + self.fAlwaysUploadLogs = False; + self.fAlwaysUploadScreenshots = False; + self.fAlwaysUploadRecordings = False; # Only upload recording files on failure by default. + self.fEnableDebugger = True; + self.adRecordingFiles = []; + self.fRecordingEnabled = False; # Don't record by default (yet). + self.fRecordingAudio = False; # Don't record audio by default. + self.cSecsRecordingMax = 0; # No recording time limit in seconds. + self.cMbRecordingMax = 195; # The test manager web server has a configured upload limit of 200 MiBs. + ## @todo Can we query the configured value here + # (via `from testmanager import config`)? + + # Drop LD_PRELOAD and enable memory leak detection in LSAN_OPTIONS from vboxinstall.py + # before doing build detection. This is a little crude and inflexible... + if 'LD_PRELOAD' in os.environ: + del os.environ['LD_PRELOAD']; + if 'LSAN_OPTIONS' in os.environ: + asLSanOptions = os.environ['LSAN_OPTIONS'].split(':'); + try: asLSanOptions.remove('detect_leaks=0'); + except: pass; + if asLSanOptions: os.environ['LSAN_OPTIONS'] = ':'.join(asLSanOptions); + else: del os.environ['LSAN_OPTIONS']; + + # Quietly detect build and validation kit. + self._detectBuild(False); + self._detectValidationKit(False); + + # Make sure all debug logs goes to the scratch area unless + # specified otherwise (more of this later on). + if 'VBOX_LOG_DEST' not in os.environ: + os.environ['VBOX_LOG_DEST'] = 'nodeny dir=%s' % (self.sScratchPath); + + + def _detectBuild(self, fQuiet = False): + """ + This is used internally to try figure a locally installed build when + running tests manually. + """ + if self.oBuild is not None: + return True; + + # Try dev build first since that's where I'll be using it first... + if True is True: # pylint: disable=comparison-with-itself + try: + self.oBuild = Build(self, None); + reporter.log('VBox %s build at %s (%s).' + % (self.oBuild.sType, self.oBuild.sInstallPath, self.oBuild.sDesignation,)); + return True; + except base.GenError: + pass; + + # Try default installation locations. + if self.sHost == 'win': + sProgFiles = os.environ.get('ProgramFiles', 'C:\\Program Files'); + asLocs = [ + os.path.join(sProgFiles, 'Oracle', 'VirtualBox'), + os.path.join(sProgFiles, 'OracleVM', 'VirtualBox'), + os.path.join(sProgFiles, 'Sun', 'VirtualBox'), + ]; + elif self.sHost == 'solaris': + asLocs = [ '/opt/VirtualBox-3.2', '/opt/VirtualBox-3.1', '/opt/VirtualBox-3.0', '/opt/VirtualBox' ]; + elif self.sHost == 'darwin': + asLocs = [ '/Applications/VirtualBox.app/Contents/MacOS' ]; + elif self.sHost == 'linux': + asLocs = [ '/opt/VirtualBox-3.2', '/opt/VirtualBox-3.1', '/opt/VirtualBox-3.0', '/opt/VirtualBox' ]; + else: + asLocs = [ '/opt/VirtualBox' ]; + if 'VBOX_INSTALL_PATH' in os.environ: + asLocs.insert(0, os.environ['VBOX_INSTALL_PATH']); + + for sLoc in asLocs: + try: + self.oBuild = Build(self, sLoc); + reporter.log('VBox %s build at %s (%s).' + % (self.oBuild.sType, self.oBuild.sInstallPath, self.oBuild.sDesignation,)); + return True; + except base.GenError: + pass; + + if not fQuiet: + reporter.error('failed to find VirtualBox installation'); + return False; + + def _detectValidationKit(self, fQuiet = False): + """ + This is used internally by the constructor to try locate an unzipped + VBox Validation Kit somewhere in the immediate proximity. + """ + if self.sVBoxValidationKit is not None: + return True; + + # + # Normally it's found where we're running from, which is the same as + # the script directly on the testboxes. + # + asCandidates = [self.sScriptPath, ]; + if g_ksValidationKitDir not in asCandidates: + asCandidates.append(g_ksValidationKitDir); + if os.getcwd() not in asCandidates: + asCandidates.append(os.getcwd()); + if self.oBuild is not None and self.oBuild.sInstallPath not in asCandidates: + asCandidates.append(self.oBuild.sInstallPath); + + # + # When working out of the tree, we'll search the current directory + # as well as parent dirs. + # + for sDir in list(asCandidates): + for i in range(10): + sDir = os.path.dirname(sDir); + if sDir not in asCandidates: + asCandidates.append(sDir); + + # + # Do the searching. + # + sCandidate = None; + for i, _ in enumerate(asCandidates): + sCandidate = asCandidates[i]; + if os.path.isfile(os.path.join(sCandidate, 'VBoxValidationKit.iso')): + break; + sCandidate = os.path.join(sCandidate, 'validationkit'); + if os.path.isfile(os.path.join(sCandidate, 'VBoxValidationKit.iso')): + break; + sCandidate = None; + + fRc = sCandidate is not None; + if fRc is False: + if not fQuiet: + reporter.error('failed to find VBox Validation Kit installation (candidates: %s)' % (asCandidates,)); + sCandidate = os.path.join(self.sScriptPath, 'validationkit'); # Don't leave the values as None. + + # + # Set the member values. + # + self.sVBoxValidationKit = sCandidate; + self.sVBoxValidationKitIso = os.path.join(sCandidate, 'VBoxValidationKit.iso'); + self.sVBoxBootSectors = os.path.join(sCandidate, 'bootsectors'); + return fRc; + + def _makeEnvironmentChanges(self): + """ + Make the necessary VBox related environment changes. + Children not importing the VBox API should call this. + """ + # Make sure we've got our own VirtualBox config and VBoxSVC (on XPCOM at least). + if not self.fUseDefaultSvc: + os.environ['VBOX_USER_HOME'] = os.path.join(self.sScratchPath, 'VBoxUserHome'); + sUser = os.environ.get('USERNAME', os.environ.get('USER', os.environ.get('LOGNAME', 'unknown'))); + os.environ['VBOX_IPC_SOCKETID'] = sUser + '-VBoxTest'; + return True; + + @staticmethod + def makeApiRevision(uMajor, uMinor, uBuild, uApiRevision): + """ Calculates an API revision number. """ + return (long(uMajor) << 56) | (long(uMinor) << 48) | (long(uBuild) << 40) | uApiRevision; + + def importVBoxApi(self): + """ + Import the 'vboxapi' module from the VirtualBox build we're using and + instantiate the two basic objects. + + This will try detect an development or installed build if no build has + been associated with the driver yet. + """ + if self.fImportedVBoxApi: + return True; + + self._makeEnvironmentChanges(); + + # Do the detecting. + self._detectBuild(); + if self.oBuild is None: + return False; + + # Avoid crashing when loading the 32-bit module (or whatever it is that goes bang). + if self.oBuild.sArch == 'x86' \ + and self.sHost == 'darwin' \ + and platform.architecture()[0] == '64bit' \ + and self.oBuild.sKind == 'development' \ + and os.getenv('VERSIONER_PYTHON_PREFER_32_BIT') != 'yes': + reporter.log("WARNING: 64-bit python on darwin, 32-bit VBox development build => crash"); + reporter.log("WARNING: bash-3.2$ /usr/bin/python2.5 ./testdriver"); + reporter.log("WARNING: or"); + reporter.log("WARNING: bash-3.2$ VERSIONER_PYTHON_PREFER_32_BIT=yes ./testdriver"); + return False; + + # Start VBoxSVC and load the vboxapi bits. + if self._startVBoxSVC() is True: + assert(self.oVBoxSvcProcess is not None); + + sSavedSysPath = sys.path; + self._setupVBoxApi(); + sys.path = sSavedSysPath; + + # Adjust the default machine folder. + if self.fImportedVBoxApi and not self.fUseDefaultSvc and self.fpApiVer >= 4.0: + sNewFolder = os.path.join(self.sScratchPath, 'VBoxUserHome', 'Machines'); + try: + self.oVBox.systemProperties.defaultMachineFolder = sNewFolder; + except: + self.fImportedVBoxApi = False; + self.oVBoxMgr = None; + self.oVBox = None; + reporter.logXcpt("defaultMachineFolder exception (sNewFolder=%s)" % (sNewFolder,)); + + # Kill VBoxSVC on failure. + if self.oVBoxMgr is None: + self._stopVBoxSVC(); + else: + assert(self.oVBoxSvcProcess is None); + return self.fImportedVBoxApi; + + def _startVBoxSVC(self): # pylint: disable=too-many-statements + """ Starts VBoxSVC. """ + assert(self.oVBoxSvcProcess is None); + + # Setup vbox logging for VBoxSVC now and start it manually. This way + # we can control both logging and shutdown. + self.sVBoxSvcLogFile = '%s/VBoxSVC-debug.log' % (self.sScratchPath,); + try: os.remove(self.sVBoxSvcLogFile); + except: pass; + os.environ['VBOX_LOG'] = self.sLogSvcGroups; + os.environ['VBOX_LOG_FLAGS'] = '%s append' % (self.sLogSvcFlags,); # Append becuse of VBoxXPCOMIPCD. + if self.sLogSvcDest: + os.environ['VBOX_LOG_DEST'] = 'nodeny ' + self.sLogSvcDest; + else: + os.environ['VBOX_LOG_DEST'] = 'nodeny file=%s' % (self.sVBoxSvcLogFile,); + os.environ['VBOXSVC_RELEASE_LOG_FLAGS'] = 'time append'; + + reporter.log2('VBoxSVC environment:'); + for sKey, sVal in sorted(os.environ.items()): + reporter.log2('%s=%s' % (sKey, sVal)); + + # Always leave a pid file behind so we can kill it during cleanup-before. + self.sVBoxSvcPidFile = '%s/VBoxSVC.pid' % (self.sScratchPath,); + fWritePidFile = True; + + cMsFudge = 1; + sVBoxSVC = '%s/VBoxSVC' % (self.oBuild.sInstallPath,); ## @todo .exe and stuff. + if self.fVBoxSvcInDebugger: + if self.sHost in ('darwin', 'freebsd', 'linux', 'solaris', ): + # Start VBoxSVC in gdb in a new terminal. + #sTerm = '/usr/bin/gnome-terminal'; - doesn't work, some fork+exec stuff confusing us. + sTerm = '/usr/bin/xterm'; + if not os.path.isfile(sTerm): sTerm = '/usr/X11/bin/xterm'; + if not os.path.isfile(sTerm): sTerm = '/usr/X11R6/bin/xterm'; + if not os.path.isfile(sTerm): sTerm = '/usr/bin/xterm'; + if not os.path.isfile(sTerm): sTerm = 'xterm'; + sGdb = '/usr/bin/gdb'; + if not os.path.isfile(sGdb): sGdb = '/usr/local/bin/gdb'; + if not os.path.isfile(sGdb): sGdb = '/usr/sfw/bin/gdb'; + if not os.path.isfile(sGdb): sGdb = 'gdb'; + sGdbCmdLine = '%s --args %s --pidfile %s' % (sGdb, sVBoxSVC, self.sVBoxSvcPidFile); + # Cool tweak to run performance analysis instead of gdb: + #sGdb = '/usr/bin/valgrind'; + #sGdbCmdLine = '%s --tool=callgrind --collect-atstart=no -- %s --pidfile %s' \ + # % (sGdb, sVBoxSVC, self.sVBoxSvcPidFile); + reporter.log('term="%s" gdb="%s"' % (sTerm, sGdbCmdLine)); + os.environ['SHELL'] = self.sOrgShell; # Non-working shell may cause gdb and/or the term problems. + ## @todo -e is deprecated; use "-- <args>". + self.oVBoxSvcProcess = base.Process.spawnp(sTerm, sTerm, '-e', sGdbCmdLine); + os.environ['SHELL'] = self.sOurShell; + if self.oVBoxSvcProcess is not None: + reporter.log('Press enter or return after starting VBoxSVC in the debugger...'); + sys.stdin.read(1); + fWritePidFile = False; + + elif self.sHost == 'win': + sWinDbg = 'c:\\Program Files\\Debugging Tools for Windows\\windbg.exe'; + if not os.path.isfile(sWinDbg): sWinDbg = 'c:\\Program Files\\Debugging Tools for Windows (x64)\\windbg.exe'; + if not os.path.isfile(sWinDbg): sWinDbg = 'c:\\Programme\\Debugging Tools for Windows\\windbg.exe'; # Localization rulez! pylint: disable=line-too-long + if not os.path.isfile(sWinDbg): sWinDbg = 'c:\\Programme\\Debugging Tools for Windows (x64)\\windbg.exe'; + if not os.path.isfile(sWinDbg): sWinDbg = 'windbg'; # WinDbg must be in the path; better than nothing. + # Assume that everything WinDbg needs is defined using the environment variables. + # See WinDbg help for more information. + reporter.log('windbg="%s"' % (sWinDbg)); + self.oVBoxSvcProcess = base.Process.spawn(sWinDbg, sWinDbg, sVBoxSVC + base.exeSuff()); + if self.oVBoxSvcProcess is not None: + reporter.log('Press enter or return after starting VBoxSVC in the debugger...'); + sys.stdin.read(1); + fWritePidFile = False; + ## @todo add a pipe interface similar to xpcom if feasible, i.e. if + # we can get actual handle values for pipes in python. + + else: + reporter.error('Port me!'); + else: # Run without a debugger attached. + if self.sHost in ('darwin', 'freebsd', 'linux', 'solaris', ): + # + # XPCOM - We can use a pipe to let VBoxSVC notify us when it's ready. + # + iPipeR, iPipeW = os.pipe(); + if hasattr(os, 'set_inheritable'): + os.set_inheritable(iPipeW, True); # pylint: disable=no-member + os.environ['NSPR_INHERIT_FDS'] = 'vboxsvc:startup-pipe:5:0x%x' % (iPipeW,); + reporter.log2("NSPR_INHERIT_FDS=%s" % (os.environ['NSPR_INHERIT_FDS'])); + + self.oVBoxSvcProcess = base.Process.spawn(sVBoxSVC, sVBoxSVC, '--auto-shutdown'); # SIGUSR1 requirement. + try: # Try make sure we get the SIGINT and not VBoxSVC. + os.setpgid(self.oVBoxSvcProcess.getPid(), 0); # pylint: disable=no-member + os.setpgid(0, 0); # pylint: disable=no-member + except: + reporter.logXcpt(); + + os.close(iPipeW); + try: + sResponse = os.read(iPipeR, 32); + except: + reporter.logXcpt(); + sResponse = None; + os.close(iPipeR); + + if hasattr(sResponse, 'decode'): + sResponse = sResponse.decode('utf-8', 'ignore'); + + if sResponse is None or sResponse.strip() != 'READY': + reporter.error('VBoxSVC failed starting up... (sResponse=%s)' % (sResponse,)); + if not self.oVBoxSvcProcess.wait(5000): + self.oVBoxSvcProcess.terminate(); + self.oVBoxSvcProcess.wait(5000); + self.oVBoxSvcProcess = None; + + elif self.sHost == 'win': + # + # Windows - Just fudge it for now. + # + cMsFudge = 2000; + self.oVBoxSvcProcess = base.Process.spawn(sVBoxSVC, sVBoxSVC); + + else: + reporter.error('Port me!'); + + # + # Enable automatic crash reporting if we succeeded. + # + if self.oVBoxSvcProcess is not None: + self.oVBoxSvcProcess.enableCrashReporting('crash/report/svc', 'crash/dump/svc'); + + # + # Wait for debugger to attach. + # + if self.oVBoxSvcProcess is not None and self.fVBoxSvcWaitForDebugger: + reporter.log('Press any key after attaching to VBoxSVC (pid %s) with a debugger...' + % (self.oVBoxSvcProcess.getPid(),)); + sys.stdin.read(1); + + # + # Fudge and pid file. + # + if self.oVBoxSvcProcess is not None and not self.oVBoxSvcProcess.wait(cMsFudge): + if fWritePidFile: + iPid = self.oVBoxSvcProcess.getPid(); + try: + oFile = utils.openNoInherit(self.sVBoxSvcPidFile, "w+"); + oFile.write('%s' % (iPid,)); + oFile.close(); + except: + reporter.logXcpt('sPidFile=%s' % (self.sVBoxSvcPidFile,)); + reporter.log('VBoxSVC PID=%u' % (iPid,)); + + # + # Finally add the task so we'll notice when it dies in a relatively timely manner. + # + self.addTask(self.oVBoxSvcProcess); + else: + self.oVBoxSvcProcess = None; + try: os.remove(self.sVBoxSvcPidFile); + except: pass; + + return self.oVBoxSvcProcess is not None; + + + def _killVBoxSVCByPidFile(self, sPidFile): + """ Kill a VBoxSVC given the pid from it's pid file. """ + + # Read the pid file. + if not os.path.isfile(sPidFile): + return False; + try: + oFile = utils.openNoInherit(sPidFile, "r"); + sPid = oFile.readline().strip(); + oFile.close(); + except: + reporter.logXcpt('sPidfile=%s' % (sPidFile,)); + return False; + + # Convert the pid to an integer and validate the range a little bit. + try: + iPid = long(sPid); + except: + reporter.logXcpt('sPidfile=%s sPid="%s"' % (sPidFile, sPid)); + return False; + if iPid <= 0: + reporter.log('negative pid - sPidfile=%s sPid="%s" iPid=%d' % (sPidFile, sPid, iPid)); + return False; + + # Take care checking that it's VBoxSVC we're about to inhume. + if base.processCheckPidAndName(iPid, "VBoxSVC") is not True: + reporter.log('Ignoring stale VBoxSVC pid file (pid=%s)' % (iPid,)); + return False; + + # Loop thru our different ways of getting VBoxSVC to terminate. + for aHow in [ [ base.sendUserSignal1, 5000, 'Dropping VBoxSVC a SIGUSR1 hint...'], \ + [ base.processInterrupt, 5000, 'Dropping VBoxSVC a SIGINT hint...'], \ + [ base.processTerminate, 7500, 'VBoxSVC is still around, killing it...'] ]: + reporter.log(aHow[2]); + if aHow[0](iPid) is True: + msStart = base.timestampMilli(); + while base.timestampMilli() - msStart < 5000 \ + and base.processExists(iPid): + time.sleep(0.2); + + fRc = not base.processExists(iPid); + if fRc is True: + break; + if fRc: + reporter.log('Successfully killed VBoxSVC (pid=%s)' % (iPid,)); + else: + reporter.log('Failed to kill VBoxSVC (pid=%s)' % (iPid,)); + return fRc; + + def _stopVBoxSVC(self): + """ + Stops VBoxSVC. Try the polite way first. + """ + + if self.oVBoxSvcProcess: + self.removeTask(self.oVBoxSvcProcess); + self.oVBoxSvcProcess.enableCrashReporting(None, None); # Disables it. + + fRc = False; + if self.oVBoxSvcProcess is not None \ + and not self.fVBoxSvcInDebugger: + # by process object. + if self.oVBoxSvcProcess.isRunning(): + reporter.log('Dropping VBoxSVC a SIGUSR1 hint...'); + if not self.oVBoxSvcProcess.sendUserSignal1() \ + or not self.oVBoxSvcProcess.wait(5000): + reporter.log('Dropping VBoxSVC a SIGINT hint...'); + if not self.oVBoxSvcProcess.interrupt() \ + or not self.oVBoxSvcProcess.wait(5000): + reporter.log('VBoxSVC is still around, killing it...'); + self.oVBoxSvcProcess.terminate(); + self.oVBoxSvcProcess.wait(7500); + else: + reporter.log('VBoxSVC is no longer running...'); + + if not self.oVBoxSvcProcess.isRunning(): + iExit = self.oVBoxSvcProcess.getExitCode(); + if iExit != 0 or not self.oVBoxSvcProcess.isNormalExit(): + reporter.error("VBoxSVC exited with status %d (%#x)" % (iExit, self.oVBoxSvcProcess.uExitCode)); + self.oVBoxSvcProcess = None; + else: + # by pid file. + self._killVBoxSVCByPidFile('%s/VBoxSVC.pid' % (self.sScratchPath,)); + return fRc; + + def _setupVBoxApi(self): + """ + Import and set up the vboxapi. + The caller saves and restores sys.path. + """ + + # Setup vbox logging for self (the test driver). + self.sSelfLogFile = '%s/VBoxTestDriver.log' % (self.sScratchPath,); + try: os.remove(self.sSelfLogFile); + except: pass; + os.environ['VBOX_LOG'] = self.sLogSelfGroups; + os.environ['VBOX_LOG_FLAGS'] = '%s append' % (self.sLogSelfFlags, ); + if self.sLogSelfDest: + os.environ['VBOX_LOG_DEST'] = 'nodeny ' + self.sLogSelfDest; + else: + os.environ['VBOX_LOG_DEST'] = 'nodeny file=%s' % (self.sSelfLogFile,); + os.environ['VBOX_RELEASE_LOG_FLAGS'] = 'time append'; + + reporter.log2('Self environment:'); + for sKey, sVal in sorted(os.environ.items()): + reporter.log2('%s=%s' % (sKey, sVal)); + + # Hack the sys.path + environment so the vboxapi can be found. + sys.path.insert(0, self.oBuild.sInstallPath); + if self.oBuild.sSdkPath is not None: + sys.path.insert(0, os.path.join(self.oBuild.sSdkPath, 'installer')) + sys.path.insert(1, os.path.join(self.oBuild.sSdkPath, 'install')); # stupid stupid windows installer! + sys.path.insert(2, os.path.join(self.oBuild.sSdkPath, 'bindings', 'xpcom', 'python')) + os.environ['VBOX_PROGRAM_PATH'] = self.oBuild.sInstallPath; + reporter.log("sys.path: %s" % (sys.path)); + + try: + from vboxapi import VirtualBoxManager; # pylint: disable=import-error + except: + reporter.logXcpt('Error importing vboxapi'); + return False; + + # Exception and error hacks. + try: + # pylint: disable=import-error + if self.sHost == 'win': + from pythoncom import com_error as NativeComExceptionClass # pylint: disable=no-name-in-module + import winerror as NativeComErrorClass + else: + from xpcom import Exception as NativeComExceptionClass + from xpcom import nsError as NativeComErrorClass + # pylint: enable=import-error + except: + reporter.logXcpt('Error importing (XP)COM related stuff for exception hacks and errors'); + return False; + __deployExceptionHacks__(NativeComExceptionClass) + ComError.copyErrors(NativeComErrorClass); + + # Create the manager. + try: + self.oVBoxMgr = VirtualBoxManager(None, None) + except: + self.oVBoxMgr = None; + reporter.logXcpt('VirtualBoxManager exception'); + return False; + + # Figure the API version. + try: + oVBox = self.oVBoxMgr.getVirtualBox(); + + try: + sVer = oVBox.version; + except: + reporter.logXcpt('Failed to get VirtualBox version, assuming 4.0.0'); + sVer = "4.0.0"; + reporter.log("IVirtualBox.version=%s" % (sVer,)); + + # Convert the string to three integer values and check ranges. + asVerComponents = sVer.split('.'); + try: + sLast = asVerComponents[2].split('_')[0].split('r')[0]; + aiVerComponents = (int(asVerComponents[0]), int(asVerComponents[1]), int(sLast)); + except: + raise base.GenError('Malformed version "%s"' % (sVer,)); + if aiVerComponents[0] < 3 or aiVerComponents[0] > 19: + raise base.GenError('Malformed version "%s" - 1st component is out of bounds 3..19: %u' + % (sVer, aiVerComponents[0])); + if aiVerComponents[1] < 0 or aiVerComponents[1] > 9: + raise base.GenError('Malformed version "%s" - 2nd component is out of bounds 0..9: %u' + % (sVer, aiVerComponents[1])); + if aiVerComponents[2] < 0 or aiVerComponents[2] > 99: + raise base.GenError('Malformed version "%s" - 3rd component is out of bounds 0..99: %u' + % (sVer, aiVerComponents[2])); + + # Convert the three integers into a floating point value. The API is stable within a + # x.y release, so the third component only indicates whether it's a stable or + # development build of the next release. + self.fpApiVer = aiVerComponents[0] + 0.1 * aiVerComponents[1]; + if aiVerComponents[2] >= 51: + if self.fpApiVer not in [6.1, 5.2, 4.3, 3.2,]: + self.fpApiVer += 0.1; + else: + self.fpApiVer = int(self.fpApiVer) + 1.0; + # fudge value to be always bigger than the nominal value (0.1 gets rounded down) + if round(self.fpApiVer, 1) > self.fpApiVer: + self.fpApiVer += sys.float_info.epsilon * self.fpApiVer / 2.0; + + try: + self.uRevision = oVBox.revision; + except: + reporter.logXcpt('Failed to get VirtualBox revision, assuming 0'); + self.uRevision = 0; + reporter.log("IVirtualBox.revision=%u" % (self.uRevision,)); + + try: + self.uApiRevision = oVBox.APIRevision; + except: + reporter.logXcpt('Failed to get VirtualBox APIRevision, faking it.'); + self.uApiRevision = self.makeApiRevision(aiVerComponents[0], aiVerComponents[1], aiVerComponents[2], 0); + reporter.log("IVirtualBox.APIRevision=%#x" % (self.uApiRevision,)); + + # Patch VBox manage to gloss over portability issues (error constants, etc). + self._patchVBoxMgr(); + + # Wrap oVBox. + from testdriver.vboxwrappers import VirtualBoxWrapper; + self.oVBox = VirtualBoxWrapper(oVBox, self.oVBoxMgr, self.fpApiVer, self); + + # Install the constant wrapping hack. + vboxcon.goHackModuleClass.oVBoxMgr = self.oVBoxMgr; # VBoxConstantWrappingHack. + vboxcon.fpApiVer = self.fpApiVer; + reporter.setComXcptFormatter(formatComOrXpComException); + + except: + self.oVBoxMgr = None; + self.oVBox = None; + reporter.logXcpt("getVirtualBox / API version exception"); + return False; + + # Done + self.fImportedVBoxApi = True; + reporter.log('Found version %s (%s)' % (self.fpApiVer, sVer)); + return True; + + def _patchVBoxMgr(self): + """ + Glosses over missing self.oVBoxMgr methods on older VBox versions. + """ + + def _xcptGetResult(oSelf, oXcpt = None): + """ See vboxapi. """ + _ = oSelf; + if oXcpt is None: oXcpt = sys.exc_info()[1]; + if sys.platform == 'win32': + import winerror; # pylint: disable=import-error + hrXcpt = oXcpt.hresult; + if hrXcpt == winerror.DISP_E_EXCEPTION: + hrXcpt = oXcpt.excepinfo[5]; + else: + hrXcpt = oXcpt.error; + return hrXcpt; + + def _xcptIsDeadInterface(oSelf, oXcpt = None): + """ See vboxapi. """ + return oSelf.xcptGetStatus(oXcpt) in [ + 0x80004004, -2147467260, # NS_ERROR_ABORT + 0x800706be, -2147023170, # NS_ERROR_CALL_FAILED (RPC_S_CALL_FAILED) + 0x800706ba, -2147023174, # RPC_S_SERVER_UNAVAILABLE. + 0x800706be, -2147023170, # RPC_S_CALL_FAILED. + 0x800706bf, -2147023169, # RPC_S_CALL_FAILED_DNE. + 0x80010108, -2147417848, # RPC_E_DISCONNECTED. + 0x800706b5, -2147023179, # RPC_S_UNKNOWN_IF + ]; + + def _xcptIsOurXcptKind(oSelf, oXcpt = None): + """ See vboxapi. """ + _ = oSelf; + if oXcpt is None: oXcpt = sys.exc_info()[1]; + if sys.platform == 'win32': + from pythoncom import com_error as NativeComExceptionClass # pylint: disable=import-error,no-name-in-module + else: + from xpcom import Exception as NativeComExceptionClass # pylint: disable=import-error + return isinstance(oXcpt, NativeComExceptionClass); + + def _xcptIsEqual(oSelf, oXcpt, hrStatus): + """ See vboxapi. """ + hrXcpt = oSelf.xcptGetResult(oXcpt); + return hrXcpt == hrStatus or hrXcpt == hrStatus - 0x100000000; # pylint: disable=consider-using-in + + def _xcptToString(oSelf, oXcpt): + """ See vboxapi. """ + _ = oSelf; + if oXcpt is None: oXcpt = sys.exc_info()[1]; + return str(oXcpt); + + def _getEnumValueName(oSelf, sEnumTypeNm, oEnumValue, fTypePrefix = False): + """ See vboxapi. """ + _ = oSelf; _ = fTypePrefix; + return '%s::%s' % (sEnumTypeNm, oEnumValue); + + # Add utilities found in newer vboxapi revision. + if not hasattr(self.oVBoxMgr, 'xcptIsDeadInterface'): + import types; + self.oVBoxMgr.xcptGetResult = types.MethodType(_xcptGetResult, self.oVBoxMgr); + self.oVBoxMgr.xcptIsDeadInterface = types.MethodType(_xcptIsDeadInterface, self.oVBoxMgr); + self.oVBoxMgr.xcptIsOurXcptKind = types.MethodType(_xcptIsOurXcptKind, self.oVBoxMgr); + self.oVBoxMgr.xcptIsEqual = types.MethodType(_xcptIsEqual, self.oVBoxMgr); + self.oVBoxMgr.xcptToString = types.MethodType(_xcptToString, self.oVBoxMgr); + if not hasattr(self.oVBoxMgr, 'getEnumValueName'): + import types; + self.oVBoxMgr.getEnumValueName = types.MethodType(_getEnumValueName, self.oVBoxMgr); + + + def _teardownVBoxApi(self): # pylint: disable=too-many-statements + """ + Drop all VBox object references and shutdown com/xpcom. + """ + if not self.fImportedVBoxApi: + return True; + import gc; + + # Drop all references we've have to COM objects. + self.aoRemoteSessions = []; + self.aoVMs = []; + self.oVBoxMgr = None; + self.oVBox = None; + vboxcon.goHackModuleClass.oVBoxMgr = None; # VBoxConstantWrappingHack. + reporter.setComXcptFormatter(None); + + # Do garbage collection to try get rid of those objects. + try: + gc.collect(); + except: + reporter.logXcpt(); + self.fImportedVBoxApi = False; + + # Check whether the python is still having any COM objects/interfaces around. + cVBoxMgrs = 0; + aoObjsLeftBehind = []; + if self.sHost == 'win': + import pythoncom; # pylint: disable=import-error + try: + cIfs = pythoncom._GetInterfaceCount(); # pylint: disable=no-member,protected-access + cObjs = pythoncom._GetGatewayCount(); # pylint: disable=no-member,protected-access + if cObjs == 0 and cIfs == 0: + reporter.log('_teardownVBoxApi: no interfaces or objects left behind.'); + else: + reporter.log('_teardownVBoxApi: Python COM still has %s objects and %s interfaces...' % ( cObjs, cIfs)); + + from win32com.client import DispatchBaseClass; # pylint: disable=import-error + for oObj in gc.get_objects(): + if isinstance(oObj, DispatchBaseClass): + reporter.log('_teardownVBoxApi: %s' % (oObj,)); + aoObjsLeftBehind.append(oObj); + elif utils.getObjectTypeName(oObj) == 'VirtualBoxManager': + reporter.log('_teardownVBoxApi: %s' % (oObj,)); + cVBoxMgrs += 1; + aoObjsLeftBehind.append(oObj); + oObj = None; + except: + reporter.logXcpt(); + + # If not being used, we can safely uninitialize COM. + if cIfs == 0 and cObjs == 0 and cVBoxMgrs == 0 and not aoObjsLeftBehind: + reporter.log('_teardownVBoxApi: Calling CoUninitialize...'); + try: pythoncom.CoUninitialize(); # pylint: disable=no-member + except: reporter.logXcpt(); + else: + reporter.log('_teardownVBoxApi: Returned from CoUninitialize.'); + else: + try: + # XPCOM doesn't crash and burn like COM if you shut it down with interfaces and objects around. + # Also, it keeps a number of internal objects and interfaces around to do its job, so shutting + # it down before we go looking for dangling interfaces is more or less required. + from xpcom import _xpcom as _xpcom; # pylint: disable=import-error,useless-import-alias + hrc = _xpcom.DeinitCOM(); + cIfs = _xpcom._GetInterfaceCount(); # pylint: disable=protected-access + cObjs = _xpcom._GetGatewayCount(); # pylint: disable=protected-access + + if cObjs == 0 and cIfs == 0: + reporter.log('_teardownVBoxApi: No XPCOM interfaces or objects active. (hrc=%#x)' % (hrc,)); + else: + reporter.log('_teardownVBoxApi: %s XPCOM objects and %s interfaces still around! (hrc=%#x)' + % (cObjs, cIfs, hrc)); + if hasattr(_xpcom, '_DumpInterfaces'): + try: _xpcom._DumpInterfaces(); # pylint: disable=protected-access + except: reporter.logXcpt('_teardownVBoxApi: _DumpInterfaces failed'); + + from xpcom.client import Component; # pylint: disable=import-error + for oObj in gc.get_objects(): + if isinstance(oObj, Component): + reporter.log('_teardownVBoxApi: %s' % (oObj,)); + aoObjsLeftBehind.append(oObj); + if utils.getObjectTypeName(oObj) == 'VirtualBoxManager': + reporter.log('_teardownVBoxApi: %s' % (oObj,)); + cVBoxMgrs += 1; + aoObjsLeftBehind.append(oObj); + oObj = None; + except: + reporter.logXcpt(); + + # Try get the referrers to (XP)COM interfaces and objects that was left behind. + for iObj in range(len(aoObjsLeftBehind)): # pylint: disable=consider-using-enumerate + try: + aoReferrers = gc.get_referrers(aoObjsLeftBehind[iObj]); + reporter.log('_teardownVBoxApi: Found %u referrers to %s:' % (len(aoReferrers), aoObjsLeftBehind[iObj],)); + for oReferrer in aoReferrers: + oMyFrame = sys._getframe(0); # pylint: disable=protected-access + if oReferrer is oMyFrame: + reporter.log('_teardownVBoxApi: - frame of this function'); + elif oReferrer is aoObjsLeftBehind: + reporter.log('_teardownVBoxApi: - aoObjsLeftBehind'); + else: + fPrinted = False; + if isinstance(oReferrer, (dict, list, tuple)): + try: + aoSubReferreres = gc.get_referrers(oReferrer); + for oSubRef in aoSubReferreres: + if not isinstance(oSubRef, list) \ + and not isinstance(oSubRef, dict) \ + and oSubRef is not oMyFrame \ + and oSubRef is not aoSubReferreres: + reporter.log('_teardownVBoxApi: - %s :: %s:' + % (utils.getObjectTypeName(oSubRef), utils.getObjectTypeName(oReferrer))); + fPrinted = True; + break; + del aoSubReferreres; + except: + reporter.logXcpt('subref'); + if not fPrinted: + reporter.log('_teardownVBoxApi: - %s:' % (utils.getObjectTypeName(oReferrer),)); + try: + import pprint; + for sLine in pprint.pformat(oReferrer, width = 130).split('\n'): + reporter.log('_teardownVBoxApi: %s' % (sLine,)); + except: + reporter.log('_teardownVBoxApi: %s' % (oReferrer,)); + except: + reporter.logXcpt(); + del aoObjsLeftBehind; + + # Force garbage collection again, just for good measure. + try: + gc.collect(); + time.sleep(0.5); # fudge factor + except: + reporter.logXcpt(); + return True; + + def _powerOffAllVms(self): + """ + Tries to power off all running VMs. + """ + for oSession in self.aoRemoteSessions: + uPid = oSession.getPid(); + if uPid is not None: + reporter.log('_powerOffAllVms: PID is %s for %s, trying to kill it.' % (uPid, oSession.sName,)); + base.processKill(uPid); + else: + reporter.log('_powerOffAllVms: No PID for %s' % (oSession.sName,)); + oSession.close(); + return None; + + + + # + # Build type, OS and arch getters. + # + + def getBuildType(self): + """ + Get the build type. + """ + if not self._detectBuild(): + return 'release'; + return self.oBuild.sType; + + def getBuildOs(self): + """ + Get the build OS. + """ + if not self._detectBuild(): + return self.sHost; + return self.oBuild.sOs; + + def getBuildArch(self): + """ + Get the build arch. + """ + if not self._detectBuild(): + return self.sHostArch; + return self.oBuild.sArch; + + def getGuestAdditionsIso(self): + """ + Get the path to the guest addition iso. + """ + if not self._detectBuild(): + return None; + return self.oBuild.sGuestAdditionsIso; + + # + # Override everything from the base class so the testdrivers don't have to + # check whether we have overridden a method or not. + # + + def showUsage(self): + rc = base.TestDriver.showUsage(self); + reporter.log(''); + reporter.log('Generic VirtualBox Options:'); + reporter.log(' --vbox-session-type <type>'); + reporter.log(' Sets the session type. Typical values are: gui, headless, sdl'); + reporter.log(' Default: %s' % (self.sSessionTypeDef)); + reporter.log(' --vrdp, --no-vrdp'); + reporter.log(' Enables VRDP, ports starting at 6000'); + reporter.log(' Default: --vrdp'); + reporter.log(' --vrdp-base-port <port>'); + reporter.log(' Sets the base for VRDP port assignments.'); + reporter.log(' Default: %s' % (self.uVrdpBasePortDef)); + reporter.log(' --vbox-default-bridged-nic <interface>'); + reporter.log(' Sets the default interface for bridged networking.'); + reporter.log(' Default: autodetect'); + reporter.log(' --vbox-use-svc-defaults'); + reporter.log(' Use default locations and files for VBoxSVC. This is useful'); + reporter.log(' for automatically configuring the test VMs for debugging.'); + reporter.log(' --vbox-log'); + reporter.log(' The VBox logger group settings for everyone.'); + reporter.log(' --vbox-log-flags'); + reporter.log(' The VBox logger flags settings for everyone.'); + reporter.log(' --vbox-log-dest'); + reporter.log(' The VBox logger destination settings for everyone.'); + reporter.log(' --vbox-self-log'); + reporter.log(' The VBox logger group settings for the testdriver.'); + reporter.log(' --vbox-self-log-flags'); + reporter.log(' The VBox logger flags settings for the testdriver.'); + reporter.log(' --vbox-self-log-dest'); + reporter.log(' The VBox logger destination settings for the testdriver.'); + reporter.log(' --vbox-session-log'); + reporter.log(' The VM session logger group settings.'); + reporter.log(' --vbox-session-log-flags'); + reporter.log(' The VM session logger flags.'); + reporter.log(' --vbox-session-log-dest'); + reporter.log(' The VM session logger destination settings.'); + reporter.log(' --vbox-svc-log'); + reporter.log(' The VBoxSVC logger group settings.'); + reporter.log(' --vbox-svc-log-flags'); + reporter.log(' The VBoxSVC logger flag settings.'); + reporter.log(' --vbox-svc-log-dest'); + reporter.log(' The VBoxSVC logger destination settings.'); + reporter.log(' --vbox-svc-debug'); + reporter.log(' Start VBoxSVC in a debugger.'); + reporter.log(' --vbox-svc-wait-debug'); + reporter.log(' Start VBoxSVC and wait for debugger to attach to it.'); + reporter.log(' --vbox-always-upload-logs'); + reporter.log(' Whether to always upload log files, or only do so on failure.'); + reporter.log(' --vbox-always-upload-screenshots'); + reporter.log(' Whether to always upload final screen shots, or only do so on failure.'); + reporter.log(' --vbox-always-upload-recordings, --no-vbox-always-upload-recordings'); + reporter.log(' Whether to always upload recordings, or only do so on failure.'); + reporter.log(' Default: --no-vbox-always-upload-recordings'); + reporter.log(' --vbox-debugger, --no-vbox-debugger'); + reporter.log(' Enables the VBox debugger, port at 5000'); + reporter.log(' Default: --vbox-debugger'); + reporter.log(' --vbox-recording, --no-vbox-recording'); + reporter.log(' Enables/disables recording.'); + reporter.log(' Default: --no-vbox-recording'); + reporter.log(' --vbox-recording-audio, --no-vbox-recording-audio'); + reporter.log(' Enables/disables audio recording.'); + reporter.log(' Default: --no-vbox-recording-audio'); + reporter.log(' --vbox-recording-max-time <seconds>'); + reporter.log(' Limits the maximum recording time in seconds.'); + reporter.log(' Default: Unlimited.'); + reporter.log(' --vbox-recording-max-file-size <MiB>'); + reporter.log(' Limits the maximum per-file size in MiB.'); + reporter.log(' Explicitly specify 0 for unlimited size.'); + reporter.log(' Default: 195 MB.'); + if self.oTestVmSet is not None: + self.oTestVmSet.showUsage(); + return rc; + + def parseOption(self, asArgs, iArg): # pylint: disable=too-many-branches,too-many-statements + if asArgs[iArg] == '--vbox-session-type': + iArg += 1; + if iArg >= len(asArgs): + raise base.InvalidOption('The "--vbox-session-type" takes an argument'); + self.sSessionType = asArgs[iArg]; + elif asArgs[iArg] == '--vrdp': + self.fEnableVrdp = True; + elif asArgs[iArg] == '--no-vrdp': + self.fEnableVrdp = False; + elif asArgs[iArg] == '--vrdp-base-port': + iArg += 1; + if iArg >= len(asArgs): + raise base.InvalidOption('The "--vrdp-base-port" takes an argument'); + try: self.uVrdpBasePort = int(asArgs[iArg]); + except: raise base.InvalidOption('The "--vrdp-base-port" value "%s" is not a valid integer' % (asArgs[iArg],)); + if self.uVrdpBasePort <= 0 or self.uVrdpBasePort >= 65530: + raise base.InvalidOption('The "--vrdp-base-port" value "%s" is not in the valid range (1..65530)' + % (asArgs[iArg],)); + elif asArgs[iArg] == '--vbox-default-bridged-nic': + iArg += 1; + if iArg >= len(asArgs): + raise base.InvalidOption('The "--vbox-default-bridged-nic" takes an argument'); + self.sDefBridgedNic = asArgs[iArg]; + elif asArgs[iArg] == '--vbox-use-svc-defaults': + self.fUseDefaultSvc = True; + elif asArgs[iArg] == '--vbox-self-log': + iArg += 1; + if iArg >= len(asArgs): + raise base.InvalidOption('The "--vbox-self-log" takes an argument'); + self.sLogSelfGroups = asArgs[iArg]; + elif asArgs[iArg] == '--vbox-self-log-flags': + iArg += 1; + if iArg >= len(asArgs): + raise base.InvalidOption('The "--vbox-self-log-flags" takes an argument'); + self.sLogSelfFlags = asArgs[iArg]; + elif asArgs[iArg] == '--vbox-self-log-dest': + iArg += 1; + if iArg >= len(asArgs): + raise base.InvalidOption('The "--vbox-self-log-dest" takes an argument'); + self.sLogSelfDest = asArgs[iArg]; + elif asArgs[iArg] == '--vbox-session-log': + iArg += 1; + if iArg >= len(asArgs): + raise base.InvalidOption('The "--vbox-session-log" takes an argument'); + self.sLogSessionGroups = asArgs[iArg]; + elif asArgs[iArg] == '--vbox-session-log-flags': + iArg += 1; + if iArg >= len(asArgs): + raise base.InvalidOption('The "--vbox-session-log-flags" takes an argument'); + self.sLogSessionFlags = asArgs[iArg]; + elif asArgs[iArg] == '--vbox-session-log-dest': + iArg += 1; + if iArg >= len(asArgs): + raise base.InvalidOption('The "--vbox-session-log-dest" takes an argument'); + self.sLogSessionDest = asArgs[iArg]; + elif asArgs[iArg] == '--vbox-svc-log': + iArg += 1; + if iArg >= len(asArgs): + raise base.InvalidOption('The "--vbox-svc-log" takes an argument'); + self.sLogSvcGroups = asArgs[iArg]; + elif asArgs[iArg] == '--vbox-svc-log-flags': + iArg += 1; + if iArg >= len(asArgs): + raise base.InvalidOption('The "--vbox-svc-log-flags" takes an argument'); + self.sLogSvcFlags = asArgs[iArg]; + elif asArgs[iArg] == '--vbox-svc-log-dest': + iArg += 1; + if iArg >= len(asArgs): + raise base.InvalidOption('The "--vbox-svc-log-dest" takes an argument'); + self.sLogSvcDest = asArgs[iArg]; + elif asArgs[iArg] == '--vbox-log': + iArg += 1; + if iArg >= len(asArgs): + raise base.InvalidOption('The "--vbox-log" takes an argument'); + self.sLogSelfGroups = asArgs[iArg]; + self.sLogSessionGroups = asArgs[iArg]; + self.sLogSvcGroups = asArgs[iArg]; + elif asArgs[iArg] == '--vbox-log-flags': + iArg += 1; + if iArg >= len(asArgs): + raise base.InvalidOption('The "--vbox-svc-flags" takes an argument'); + self.sLogSelfFlags = asArgs[iArg]; + self.sLogSessionFlags = asArgs[iArg]; + self.sLogSvcFlags = asArgs[iArg]; + elif asArgs[iArg] == '--vbox-log-dest': + iArg += 1; + if iArg >= len(asArgs): + raise base.InvalidOption('The "--vbox-log-dest" takes an argument'); + self.sLogSelfDest = asArgs[iArg]; + self.sLogSessionDest = asArgs[iArg]; + self.sLogSvcDest = asArgs[iArg]; + elif asArgs[iArg] == '--vbox-svc-debug': + self.fVBoxSvcInDebugger = True; + elif asArgs[iArg] == '--vbox-svc-wait-debug': + self.fVBoxSvcWaitForDebugger = True; + elif asArgs[iArg] == '--vbox-always-upload-logs': + self.fAlwaysUploadLogs = True; + elif asArgs[iArg] == '--vbox-always-upload-screenshots': + self.fAlwaysUploadScreenshots = True; + elif asArgs[iArg] == '--no-vbox-always-upload-recordings': + self.fAlwaysUploadRecordings = False; + elif asArgs[iArg] == '--vbox-always-upload-recordings': + self.fAlwaysUploadRecordings = True; + elif asArgs[iArg] == '--vbox-debugger': + self.fEnableDebugger = True; + elif asArgs[iArg] == '--no-vbox-debugger': + self.fEnableDebugger = False; + elif asArgs[iArg] == '--vbox-recording': + self.fRecordingEnabled = True; + elif asArgs[iArg] == '--vbox-no-recording': + self.fRecordingEnabled = False; + elif asArgs[iArg] == '--no-vbox-recording-audio': + self.fRecordingAudio = False; + elif asArgs[iArg] == '--vbox-recording-audio': + self.fRecordingAudio = True; + elif asArgs[iArg] == '--vbox-recording-max-time': + iArg += 1; + if iArg >= len(asArgs): + raise base.InvalidOption('The "--vbox-recording-max-time" takes an argument'); + self.cSecsRecordingMax = int(asArgs[iArg]); + elif asArgs[iArg] == '--vbox-recording-max-file-size': + iArg += 1; + if iArg >= len(asArgs): + raise base.InvalidOption('The "--vbox-recording-max-file-size" takes an argument'); + self.cMbRecordingMax = int(asArgs[iArg]); + else: + # Relevant for selecting VMs to test? + if self.oTestVmSet is not None: + iRc = self.oTestVmSet.parseOption(asArgs, iArg); + if iRc != iArg: + return iRc; + + # Hand it to the base class. + return base.TestDriver.parseOption(self, asArgs, iArg); + return iArg + 1; + + def completeOptions(self): + return base.TestDriver.completeOptions(self); + + def getNetworkAdapterNameFromType(self, oNic): + """ + Returns the network adapter name from a given adapter type. + + Returns an empty string if not found / invalid. + """ + sAdpName = ''; + if oNic.adapterType in (vboxcon.NetworkAdapterType_Am79C970A, \ + vboxcon.NetworkAdapterType_Am79C973, \ + vboxcon.NetworkAdapterType_Am79C960): + sAdpName = 'pcnet'; + elif oNic.adapterType in (vboxcon.NetworkAdapterType_I82540EM, \ + vboxcon.NetworkAdapterType_I82543GC, \ + vboxcon.NetworkAdapterType_I82545EM): + sAdpName = 'e1000'; + elif oNic.adapterType == vboxcon.NetworkAdapterType_Virtio: + sAdpName = 'virtio-net'; + return sAdpName; + + def getResourceSet(self): + asRsrcs = []; + if self.oTestVmSet is not None: + asRsrcs.extend(self.oTestVmSet.getResourceSet()); + asRsrcs.extend(base.TestDriver.getResourceSet(self)); + return asRsrcs; + + def actionExtract(self): + return base.TestDriver.actionExtract(self); + + def actionVerify(self): + return base.TestDriver.actionVerify(self); + + def actionConfig(self): + return base.TestDriver.actionConfig(self); + + def actionExecute(self): + return base.TestDriver.actionExecute(self); + + def actionCleanupBefore(self): + """ + Kill any VBoxSVC left behind by a previous test run. + """ + self._killVBoxSVCByPidFile('%s/VBoxSVC.pid' % (self.sScratchPath,)); + return base.TestDriver.actionCleanupBefore(self); + + def actionCleanupAfter(self): + """ + Clean up the VBox bits and then call the base driver. + + If your test driver overrides this, it should normally call us at the + end of the job. + """ + cErrorsEntry = reporter.getErrorCount(); + + # Kill any left over VM processes. + self._powerOffAllVms(); + + # Drop all VBox object references and shutdown xpcom then + # terminating VBoxSVC, with extreme prejudice if need be. + self._teardownVBoxApi(); + self._stopVBoxSVC(); + + # Add the VBoxSVC and testdriver debug+release log files. + if self.fAlwaysUploadLogs or reporter.getErrorCount() > 0: + if self.sVBoxSvcLogFile is not None and os.path.isfile(self.sVBoxSvcLogFile): + reporter.addLogFile(self.sVBoxSvcLogFile, 'log/debug/svc', 'Debug log file for VBoxSVC'); + self.sVBoxSvcLogFile = None; + + if self.sSelfLogFile is not None and os.path.isfile(self.sSelfLogFile): + reporter.addLogFile(self.sSelfLogFile, 'log/debug/client', 'Debug log file for the test driver'); + self.sSelfLogFile = None; + + if self.sSessionLogFile is not None and os.path.isfile(self.sSessionLogFile): + reporter.addLogFile(self.sSessionLogFile, 'log/debug/session', 'Debug log file for the VM session'); + self.sSessionLogFile = None; + + sVBoxSvcRelLog = os.path.join(self.sScratchPath, 'VBoxUserHome', 'VBoxSVC.log'); + if os.path.isfile(sVBoxSvcRelLog): + reporter.addLogFile(sVBoxSvcRelLog, 'log/release/svc', 'Release log file for VBoxSVC'); + for sSuff in [ '.1', '.2', '.3', '.4', '.5', '.6', '.7', '.8' ]: + if os.path.isfile(sVBoxSvcRelLog + sSuff): + reporter.addLogFile(sVBoxSvcRelLog + sSuff, 'log/release/svc', 'Release log file for VBoxSVC'); + + # Finally, call the base driver to wipe the scratch space. + fRc = base.TestDriver.actionCleanupAfter(self); + + # Flag failure if the error count increased. + if reporter.getErrorCount() > cErrorsEntry: + fRc = False; + return fRc; + + + def actionAbort(self): + """ + Terminate VBoxSVC if we've got a pid file. + """ + # + # Take default action first, then kill VBoxSVC. The other way around + # is problematic since the testscript would continue running and possibly + # trigger a new VBoxSVC to start. + # + fRc1 = base.TestDriver.actionAbort(self); + fRc2 = self._killVBoxSVCByPidFile('%s/VBoxSVC.pid' % (self.sScratchPath,)); + return fRc1 is True and fRc2 is True; + + def onExit(self, iRc): + """ + Stop VBoxSVC if we've started it. + """ + if self.oVBoxSvcProcess is not None: + reporter.log('*** Shutting down the VBox API... (iRc=%s)' % (iRc,)); + self._powerOffAllVms(); + self._teardownVBoxApi(); + self._stopVBoxSVC(); + reporter.log('*** VBox API shutdown done.'); + return base.TestDriver.onExit(self, iRc); + + + # + # Task wait method override. + # + + def notifyAboutReadyTask(self, oTask): + """ + Overriding base.TestDriver.notifyAboutReadyTask. + """ + try: + self.oVBoxMgr.interruptWaitEvents(); + reporter.log2('vbox.notifyAboutReadyTask: called interruptWaitEvents'); + except: + reporter.logXcpt('vbox.notifyAboutReadyTask'); + return base.TestDriver.notifyAboutReadyTask(self, oTask); + + def waitForTasksSleepWorker(self, cMsTimeout): + """ + Overriding base.TestDriver.waitForTasksSleepWorker. + """ + try: + rc = self.oVBoxMgr.waitForEvents(int(cMsTimeout)); + _ = rc; #reporter.log2('vbox.waitForTasksSleepWorker(%u): true (waitForEvents -> %s)' % (cMsTimeout, rc)); + reporter.doPollWork('vbox.TestDriver.waitForTasksSleepWorker'); + return True; + except KeyboardInterrupt: + raise; + except: + reporter.logXcpt('vbox.waitForTasksSleepWorker'); + return False; + + # + # Utility methods. + # + + def processEvents(self, cMsTimeout = 0): + """ + Processes events, returning after the first batch has been processed + or the time limit has been reached. + + Only Ctrl-C exception, no return. + """ + try: + self.oVBoxMgr.waitForEvents(cMsTimeout); + except KeyboardInterrupt: + raise; + except: + pass; + return None; + + def processPendingEvents(self): + """ processEvents(0) - no waiting. """ + return self.processEvents(0); + + def sleep(self, cSecs): + """ + Sleep for a specified amount of time, processing XPCOM events all the while. + """ + cMsTimeout = long(cSecs * 1000); + msStart = base.timestampMilli(); + self.processEvents(0); + while True: + cMsElapsed = base.timestampMilli() - msStart; + if cMsElapsed > cMsTimeout: + break; + #reporter.log2('cMsTimeout=%s - cMsElapsed=%d => %s' % (cMsTimeout, cMsElapsed, cMsTimeout - cMsElapsed)); + self.processEvents(cMsTimeout - cMsElapsed); + return None; + + def _logVmInfoUnsafe(self, oVM): # pylint: disable=too-many-statements,too-many-branches + """ + Internal worker for logVmInfo that is wrapped in try/except. + """ + reporter.log(" Name: %s" % (oVM.name,)); + reporter.log(" ID: %s" % (oVM.id,)); + oOsType = self.oVBox.getGuestOSType(oVM.OSTypeId); + reporter.log(" OS Type: %s - %s" % (oVM.OSTypeId, oOsType.description,)); + reporter.log(" Machine state: %s" % (oVM.state,)); + reporter.log(" Session state: %s" % (oVM.sessionState,)); + if self.fpApiVer >= 4.2: + reporter.log(" Session PID: %u (%#x)" % (oVM.sessionPID, oVM.sessionPID,)); + else: + reporter.log(" Session PID: %u (%#x)" % (oVM.sessionPid, oVM.sessionPid,)); + if self.fpApiVer >= 5.0: + reporter.log(" Session Name: %s" % (oVM.sessionName,)); + else: + reporter.log(" Session Name: %s" % (oVM.sessionType,)); + reporter.log(" CPUs: %s" % (oVM.CPUCount,)); + reporter.log(" RAM: %sMB" % (oVM.memorySize,)); + if self.fpApiVer >= 6.1 and hasattr(oVM, 'graphicsAdapter'): + reporter.log(" VRAM: %sMB" % (oVM.graphicsAdapter.VRAMSize,)); + reporter.log(" Monitors: %s" % (oVM.graphicsAdapter.monitorCount,)); + reporter.log(" GraphicsController: %s" + % (self.oVBoxMgr.getEnumValueName('GraphicsControllerType', # pylint: disable=not-callable + oVM.graphicsAdapter.graphicsControllerType),)); + else: + reporter.log(" VRAM: %sMB" % (oVM.VRAMSize,)); + reporter.log(" Monitors: %s" % (oVM.monitorCount,)); + reporter.log(" GraphicsController: %s" + % (self.oVBoxMgr.getEnumValueName('GraphicsControllerType', oVM.graphicsControllerType),)); # pylint: disable=not-callable + reporter.log(" Chipset: %s" % (self.oVBoxMgr.getEnumValueName('ChipsetType', oVM.chipsetType),)); # pylint: disable=not-callable + if self.fpApiVer >= 6.2 and hasattr(vboxcon, 'IommuType_None'): + reporter.log(" IOMMU: %s" % (self.oVBoxMgr.getEnumValueName('IommuType', oVM.iommuType),)); # pylint: disable=not-callable + reporter.log(" Firmware: %s" % (self.oVBoxMgr.getEnumValueName('FirmwareType', oVM.firmwareType),)); # pylint: disable=not-callable + reporter.log(" HwVirtEx: %s" % (oVM.getHWVirtExProperty(vboxcon.HWVirtExPropertyType_Enabled),)); + reporter.log(" VPID support: %s" % (oVM.getHWVirtExProperty(vboxcon.HWVirtExPropertyType_VPID),)); + reporter.log(" Nested paging: %s" % (oVM.getHWVirtExProperty(vboxcon.HWVirtExPropertyType_NestedPaging),)); + atTypes = [ + ( 'CPUPropertyType_PAE', 'PAE: '), + ( 'CPUPropertyType_LongMode', 'Long-mode: '), + ( 'CPUPropertyType_HWVirt', 'Nested VT-x/AMD-V: '), + ( 'CPUPropertyType_APIC', 'APIC: '), + ( 'CPUPropertyType_X2APIC', 'X2APIC: '), + ( 'CPUPropertyType_TripleFaultReset', 'TripleFaultReset: '), + ( 'CPUPropertyType_IBPBOnVMExit', 'IBPBOnVMExit: '), + ( 'CPUPropertyType_SpecCtrl', 'SpecCtrl: '), + ( 'CPUPropertyType_SpecCtrlByHost', 'SpecCtrlByHost: '), + ]; + for sEnumValue, sDesc in atTypes: + if hasattr(vboxcon, sEnumValue): + reporter.log(" %s%s" % (sDesc, oVM.getCPUProperty(getattr(vboxcon, sEnumValue)),)); + reporter.log(" ACPI: %s" % (oVM.BIOSSettings.ACPIEnabled,)); + reporter.log(" IO-APIC: %s" % (oVM.BIOSSettings.IOAPICEnabled,)); + if self.fpApiVer >= 3.2: + if self.fpApiVer >= 4.2: + reporter.log(" HPET: %s" % (oVM.HPETEnabled,)); + else: + reporter.log(" HPET: %s" % (oVM.hpetEnabled,)); + if self.fpApiVer >= 6.1 and hasattr(oVM, 'graphicsAdapter'): + reporter.log(" 3D acceleration: %s" % (oVM.graphicsAdapter.accelerate3DEnabled,)); + reporter.log(" 2D acceleration: %s" % (oVM.graphicsAdapter.accelerate2DVideoEnabled,)); + else: + reporter.log(" 3D acceleration: %s" % (oVM.accelerate3DEnabled,)); + reporter.log(" 2D acceleration: %s" % (oVM.accelerate2DVideoEnabled,)); + reporter.log(" TeleporterEnabled: %s" % (oVM.teleporterEnabled,)); + reporter.log(" TeleporterPort: %s" % (oVM.teleporterPort,)); + reporter.log(" TeleporterAddress: %s" % (oVM.teleporterAddress,)); + reporter.log(" TeleporterPassword: %s" % (oVM.teleporterPassword,)); + reporter.log(" Clipboard mode: %s" % (oVM.clipboardMode,)); + if self.fpApiVer >= 5.0: + reporter.log(" Drag and drop mode: %s" % (oVM.dnDMode,)); + elif self.fpApiVer >= 4.3: + reporter.log(" Drag and drop mode: %s" % (oVM.dragAndDropMode,)); + if self.fpApiVer >= 4.0: + reporter.log(" VRDP server: %s" % (oVM.VRDEServer.enabled,)); + try: sPorts = oVM.VRDEServer.getVRDEProperty("TCP/Ports"); + except: sPorts = ""; + reporter.log(" VRDP server ports: %s" % (sPorts,)); + reporter.log(" VRDP auth: %s (%s)" % (oVM.VRDEServer.authType, oVM.VRDEServer.authLibrary,)); + else: + reporter.log(" VRDP server: %s" % (oVM.VRDPServer.enabled,)); + reporter.log(" VRDP server ports: %s" % (oVM.VRDPServer.ports,)); + reporter.log(" Last changed: %s" % (oVM.lastStateChange,)); + + aoControllers = self.oVBoxMgr.getArray(oVM, 'storageControllers') + if aoControllers: + reporter.log(" Controllers:"); + for oCtrl in aoControllers: + reporter.log(" %s %s bus: %s type: %s" % (oCtrl.name, oCtrl.controllerType, oCtrl.bus, oCtrl.controllerType,)); + if self.fpApiVer >= 7.0: + oAdapter = oVM.audioSettings.adapter; + else: + oAdapter = oVM.audioAdapter; + reporter.log(" AudioController: %s" + % (self.oVBoxMgr.getEnumValueName('AudioControllerType', oAdapter.audioController),)); # pylint: disable=not-callable + reporter.log(" AudioEnabled: %s" % (oAdapter.enabled,)); + reporter.log(" Host AudioDriver: %s" + % (self.oVBoxMgr.getEnumValueName('AudioDriverType', oAdapter.audioDriver),)); # pylint: disable=not-callable + + self.processPendingEvents(); + aoAttachments = self.oVBoxMgr.getArray(oVM, 'mediumAttachments') + if aoAttachments: + reporter.log(" Attachments:"); + for oAtt in aoAttachments: + sCtrl = "Controller: %s port: %s device: %s type: %s" % (oAtt.controller, oAtt.port, oAtt.device, oAtt.type); + oMedium = oAtt.medium + if oAtt.type == vboxcon.DeviceType_HardDisk: + reporter.log(" %s: HDD" % sCtrl); + reporter.log(" Id: %s" % (oMedium.id,)); + reporter.log(" Name: %s" % (oMedium.name,)); + reporter.log(" Format: %s" % (oMedium.format,)); + reporter.log(" Location: %s" % (oMedium.location,)); + + if oAtt.type == vboxcon.DeviceType_DVD: + reporter.log(" %s: DVD" % sCtrl); + if oMedium: + reporter.log(" Id: %s" % (oMedium.id,)); + reporter.log(" Name: %s" % (oMedium.name,)); + if oMedium.hostDrive: + reporter.log(" Host DVD %s" % (oMedium.location,)); + if oAtt.passthrough: + reporter.log(" [passthrough mode]"); + else: + reporter.log(" Virtual image: %s" % (oMedium.location,)); + reporter.log(" Size: %s" % (oMedium.size,)); + else: + reporter.log(" empty"); + + if oAtt.type == vboxcon.DeviceType_Floppy: + reporter.log(" %s: Floppy" % sCtrl); + if oMedium: + reporter.log(" Id: %s" % (oMedium.id,)); + reporter.log(" Name: %s" % (oMedium.name,)); + if oMedium.hostDrive: + reporter.log(" Host floppy: %s" % (oMedium.location,)); + else: + reporter.log(" Virtual image: %s" % (oMedium.location,)); + reporter.log(" Size: %s" % (oMedium.size,)); + else: + reporter.log(" empty"); + self.processPendingEvents(); + + reporter.log(" Network Adapter:"); + for iSlot in range(0, 32): + try: oNic = oVM.getNetworkAdapter(iSlot) + except: break; + if not oNic.enabled: + reporter.log2(" slot #%d found but not enabled, skipping" % (iSlot,)); + continue; + reporter.log(" slot #%d: type: %s (%s) MAC Address: %s lineSpeed: %s" + % (iSlot, self.oVBoxMgr.getEnumValueName('NetworkAdapterType', oNic.adapterType), # pylint: disable=not-callable + oNic.adapterType, oNic.MACAddress, oNic.lineSpeed) ); + + if oNic.attachmentType == vboxcon.NetworkAttachmentType_NAT: + reporter.log(" attachmentType: NAT (%s)" % (oNic.attachmentType,)); + if self.fpApiVer >= 4.1: + reporter.log(" nat-network: %s" % (oNic.NATNetwork,)); + if self.fpApiVer >= 7.0 and hasattr(oNic.NATEngine, 'localhostReachable'): + reporter.log(" localhostReachable: %s" % (oNic.NATEngine.localhostReachable,)); + + elif oNic.attachmentType == vboxcon.NetworkAttachmentType_Bridged: + reporter.log(" attachmentType: Bridged (%s)" % (oNic.attachmentType,)); + if self.fpApiVer >= 4.1: + reporter.log(" hostInterface: %s" % (oNic.bridgedInterface,)); + else: + reporter.log(" hostInterface: %s" % (oNic.hostInterface,)); + elif oNic.attachmentType == vboxcon.NetworkAttachmentType_Internal: + reporter.log(" attachmentType: Internal (%s)" % (oNic.attachmentType,)); + reporter.log(" intnet-name: %s" % (oNic.internalNetwork,)); + elif oNic.attachmentType == vboxcon.NetworkAttachmentType_HostOnly: + reporter.log(" attachmentType: HostOnly (%s)" % (oNic.attachmentType,)); + if self.fpApiVer >= 4.1: + reporter.log(" hostInterface: %s" % (oNic.hostOnlyInterface,)); + else: + reporter.log(" hostInterface: %s" % (oNic.hostInterface,)); + else: + if self.fpApiVer >= 7.0: + if oNic.attachmentType == vboxcon.NetworkAttachmentType_HostOnlyNetwork: + reporter.log(" attachmentType: HostOnlyNetwork (%s)" % (oNic.attachmentType,)); + reporter.log(" hostonly-net: %s" % (oNic.hostOnlyNetwork,)); + elif self.fpApiVer >= 4.1: + if oNic.attachmentType == vboxcon.NetworkAttachmentType_Generic: + reporter.log(" attachmentType: Generic (%s)" % (oNic.attachmentType,)); + reporter.log(" generic-driver: %s" % (oNic.GenericDriver,)); + else: + reporter.log(" attachmentType: unknown-%s" % (oNic.attachmentType,)); + else: + reporter.log(" attachmentType: unknown-%s" % (oNic.attachmentType,)); + if oNic.traceEnabled: + reporter.log(" traceFile: %s" % (oNic.traceFile,)); + self.processPendingEvents(); + + reporter.log(" Serial ports:"); + for iSlot in range(0, 8): + try: oPort = oVM.getSerialPort(iSlot) + except: break; + if oPort is not None and oPort.enabled: + enmHostMode = oPort.hostMode; + reporter.log(" slot #%d: hostMode: %s (%s) I/O port: %s IRQ: %s server: %s path: %s" % + (iSlot, self.oVBoxMgr.getEnumValueName('PortMode', enmHostMode), # pylint: disable=not-callable + enmHostMode, oPort.IOBase, oPort.IRQ, oPort.server, oPort.path,) ); + self.processPendingEvents(); + + return True; + + def logVmInfo(self, oVM): # pylint: disable=too-many-statements,too-many-branches + """ + Logs VM configuration details. + + This is copy, past, search, replace and edit of infoCmd from vboxshell.py. + """ + try: + fRc = self._logVmInfoUnsafe(oVM); + except: + reporter.logXcpt(); + fRc = False; + return fRc; + + def logVmInfoByName(self, sName): + """ + logVmInfo + getVmByName. + """ + return self.logVmInfo(self.getVmByName(sName)); + + def tryFindGuestOsId(self, sIdOrDesc): + """ + Takes a guest OS ID or Description and returns the ID. + If nothing matching it is found, the input is returned unmodified. + """ + + if self.fpApiVer >= 4.0: + if sIdOrDesc == 'Solaris (64 bit)': + sIdOrDesc = 'Oracle Solaris 10 5/09 and earlier (64 bit)'; + + try: + aoGuestTypes = self.oVBoxMgr.getArray(self.oVBox, 'GuestOSTypes'); + except: + reporter.logXcpt(); + else: + for oGuestOS in aoGuestTypes: + try: + sId = oGuestOS.id; + sDesc = oGuestOS.description; + except: + reporter.logXcpt(); + else: + if sIdOrDesc in (sId, sDesc,): + sIdOrDesc = sId; + break; + self.processPendingEvents(); + return sIdOrDesc + + def resourceFindVmHd(self, sVmName, sFlavor): + """ + Search the test resources for the most recent VM HD. + + Returns path relative to the test resource root. + """ + ## @todo implement a proper search algo here. + return '4.2/' + sFlavor + '/' + sVmName + '/t-' + sVmName + '.vdi'; + + + # + # VM Api wrappers that logs errors, hides exceptions and other details. + # + + def createTestVMOnly(self, sName, sKind): + """ + Creates and register a test VM without doing any kind of configuration. + + Returns VM object (IMachine) on success, None on failure. + """ + if not self.importVBoxApi(): + return None; + + # create + register the VM + try: + if self.fpApiVer >= 7.0: # Introduces VM encryption (three new parameters, empty for now). + oVM = self.oVBox.createMachine("", sName, [], self.tryFindGuestOsId(sKind), "", "", "", ""); + elif self.fpApiVer >= 4.2: # Introduces grouping (third parameter, empty for now). + oVM = self.oVBox.createMachine("", sName, [], self.tryFindGuestOsId(sKind), ""); + elif self.fpApiVer >= 4.0: + oVM = self.oVBox.createMachine("", sName, self.tryFindGuestOsId(sKind), "", False); + elif self.fpApiVer >= 3.2: + oVM = self.oVBox.createMachine(sName, self.tryFindGuestOsId(sKind), "", "", False); + else: + oVM = self.oVBox.createMachine(sName, self.tryFindGuestOsId(sKind), "", ""); + try: + oVM.saveSettings(); + try: + self.oVBox.registerMachine(oVM); + return oVM; + except: + reporter.logXcpt(); + raise; + except: + reporter.logXcpt(); + if self.fpApiVer >= 4.0: + try: + if self.fpApiVer >= 4.3: + oProgress = oVM.deleteConfig([]); + else: + oProgress = oVM.delete(None); + self.waitOnProgress(oProgress); + except: + reporter.logXcpt(); + else: + try: oVM.deleteSettings(); + except: reporter.logXcpt(); + raise; + except: + reporter.errorXcpt('failed to create vm "%s"' % (sName)); + return None; + + # pylint: disable=too-many-arguments,too-many-locals,too-many-statements,too-many-branches + def createTestVM(self, + sName, + iGroup, + sHd = None, + cMbRam = None, + cCpus = 1, + fVirtEx = None, + fNestedPaging = None, + sDvdImage = None, + sKind = "Other", + fIoApic = None, + fNstHwVirt = None, + fPae = None, + fFastBootLogo = True, + eNic0Type = None, + eNic0AttachType = None, + sNic0NetName = 'default', + sNic0MacAddr = 'grouped', + sFloppy = None, + fNatForwardingForTxs = None, + sHddControllerType = 'IDE Controller', + fVmmDevTestingPart = None, + fVmmDevTestingMmio = False, + sFirmwareType = 'bios', + sChipsetType = 'piix3', + sIommuType = 'none', + sDvdControllerType = 'IDE Controller', + sCom1RawFile = None): + """ + Creates a test VM with a immutable HD from the test resources. + """ + # create + register the VM + oVM = self.createTestVMOnly(sName, sKind); + if not oVM: + return None; + + # Configure the VM. + fRc = True; + oSession = self.openSession(oVM); + if oSession is not None: + fRc = oSession.setupPreferredConfig(); + + if fRc and cMbRam is not None : + fRc = oSession.setRamSize(cMbRam); + if fRc and cCpus is not None: + fRc = oSession.setCpuCount(cCpus); + if fRc and fVirtEx is not None: + fRc = oSession.enableVirtEx(fVirtEx); + if fRc and fNestedPaging is not None: + fRc = oSession.enableNestedPaging(fNestedPaging); + if fRc and fIoApic is not None: + fRc = oSession.enableIoApic(fIoApic); + if fRc and fNstHwVirt is not None: + fRc = oSession.enableNestedHwVirt(fNstHwVirt); + if fRc and fPae is not None: + fRc = oSession.enablePae(fPae); + if fRc and sDvdImage is not None: + fRc = oSession.attachDvd(sDvdImage, sDvdControllerType); + if fRc and sHd is not None: + fRc = oSession.attachHd(sHd, sHddControllerType); + if fRc and sFloppy is not None: + fRc = oSession.attachFloppy(sFloppy); + if fRc and eNic0Type is not None: + fRc = oSession.setNicType(eNic0Type, 0); + if fRc and (eNic0AttachType is not None or (sNic0NetName is not None and sNic0NetName != 'default')): + fRc = oSession.setNicAttachment(eNic0AttachType, sNic0NetName, 0); + if fRc and sNic0MacAddr is not None: + if sNic0MacAddr == 'grouped': + sNic0MacAddr = '%02X' % (iGroup); + fRc = oSession.setNicMacAddress(sNic0MacAddr, 0); + # Needed to reach the host (localhost) from the guest. See xTracker #9896. + if fRc and self.fpApiVer >= 7.0: + fRc = oSession.setNicLocalhostReachable(True, 0); + if fRc and fNatForwardingForTxs is True: + fRc = oSession.setupNatForwardingForTxs(); + if fRc and fFastBootLogo is not None: + fRc = oSession.setupBootLogo(fFastBootLogo); + if fRc and self.fEnableVrdp: + fRc = oSession.setupVrdp(True, self.uVrdpBasePort + iGroup); + if fRc and fVmmDevTestingPart is not None: + fRc = oSession.enableVmmDevTestingPart(fVmmDevTestingPart, fVmmDevTestingMmio); + if fRc and sFirmwareType == 'bios': + fRc = oSession.setFirmwareType(vboxcon.FirmwareType_BIOS); + elif fRc and sFirmwareType == 'efi': + fRc = oSession.setFirmwareType(vboxcon.FirmwareType_EFI); + if fRc and self.fEnableDebugger: + fRc = oSession.setExtraData('VBoxInternal/DBGC/Enabled', '1'); + if fRc and self.fRecordingEnabled: + try: + if self.fpApiVer >= 6.1: # Only for VBox 6.1 and up now. + reporter.log('Recording enabled'); + if self.cSecsRecordingMax > 0: + reporter.log('Recording time limit is set to %d seconds' % (self.cSecsRecordingMax)); + if self.cMbRecordingMax > 0: + reporter.log('Recording file limit is set to %d MB' % (self.cMbRecordingMax)); + oRecSettings = oSession.o.machine.recordingSettings; + oRecSettings.enabled = True; + aoScreens = self.oVBoxMgr.getArray(oRecSettings, 'screens'); + for oScreen in aoScreens: + try: + oScreen.enabled = True; + sRecFile = os.path.join(self.sScratchPath, "recording-%s.webm" % (sName)); + oScreen.filename = sRecFile; + sRecFile = oScreen.filename; # Get back the file from Main, in case it was modified somehow. + dRecFile = { 'id' : oScreen.id, 'file' : sRecFile }; + self.adRecordingFiles.append(dRecFile); + if self.fpApiVer >= 7.0: + aFeatures = [ vboxcon.RecordingFeature_Video, ]; + if self.fRecordingAudio: + aFeatures.append(vboxcon.RecordingFeature_Audio); + try: + oScreen.setFeatures(aFeatures); + except: ## @todo Figure out why this is needed on Windows. + oScreen.features = aFeatures; + else: # <= VBox 6.1 the feature were kept as a ULONG. + uFeatures = vboxcon.RecordingFeature_Video; + if self.fRecordingAudio: + uFeatures = uFeatures | vboxcon.RecordingFeature_Audio; + oScreen.features = uFeatures; + reporter.log2('Recording screen %d to "%s"' % (dRecFile['id'], dRecFile['file'],)); + oScreen.maxTime = self.cSecsRecordingMax; + oScreen.maxFileSize = self.cMbRecordingMax; + except: + reporter.errorXcpt('failed to configure recording for "%s" (screen %d)' % (sName, oScreen.id)); + else: + # Not fatal. + reporter.log('Recording only available for VBox >= 6.1, sorry!') + except: + reporter.errorXcpt('failed to configure recording for "%s"' % (sName)); + if fRc and sChipsetType == 'piix3': + fRc = oSession.setChipsetType(vboxcon.ChipsetType_PIIX3); + elif fRc and sChipsetType == 'ich9': + fRc = oSession.setChipsetType(vboxcon.ChipsetType_ICH9); + if fRc and sCom1RawFile: + fRc = oSession.setupSerialToRawFile(0, sCom1RawFile); + if fRc and self.fpApiVer >= 6.2 and hasattr(vboxcon, 'IommuType_AMD') and sIommuType == 'amd': + fRc = oSession.setIommuType(vboxcon.IommuType_AMD); + elif fRc and self.fpApiVer >= 6.2 and hasattr(vboxcon, 'IommuType_Intel') and sIommuType == 'intel': + fRc = oSession.setIommuType(vboxcon.IommuType_Intel); + + if fRc: fRc = oSession.saveSettings(); + if not fRc: oSession.discardSettings(True); + oSession.close(); + if not fRc: + if self.fpApiVer >= 4.0: + try: oVM.unregister(vboxcon.CleanupMode_Full); + except: reporter.logXcpt(); + try: + if self.fpApiVer >= 4.3: + oProgress = oVM.deleteConfig([]); + else: + oProgress = oVM.delete([]); + self.waitOnProgress(oProgress); + except: + reporter.logXcpt(); + else: + try: self.oVBox.unregisterMachine(oVM.id); + except: reporter.logXcpt(); + try: oVM.deleteSettings(); + except: reporter.logXcpt(); + return None; + + # success. + reporter.log('created "%s" with name "%s"' % (oVM.id, sName)); + self.aoVMs.append(oVM); + self.logVmInfo(oVM); # testing... + return oVM; + # pylint: enable=too-many-arguments,too-many-locals,too-many-statements + + def createTestVmWithDefaults(self, # pylint: disable=too-many-arguments + sName, + iGroup, + sKind, + sDvdImage = None, + fFastBootLogo = True, + eNic0AttachType = None, + sNic0NetName = 'default', + sNic0MacAddr = 'grouped', + fVmmDevTestingPart = None, + fVmmDevTestingMmio = False, + sCom1RawFile = None): + """ + Creates a test VM with all defaults and no HDs. + """ + # create + register the VM + oVM = self.createTestVMOnly(sName, sKind); + if oVM is not None: + # Configure the VM with defaults according to sKind. + fRc = True; + oSession = self.openSession(oVM); + if oSession is not None: + if self.fpApiVer >= 6.0: + try: + oSession.o.machine.applyDefaults(''); + except: + reporter.errorXcpt('failed to apply defaults to vm "%s"' % (sName,)); + fRc = False; + else: + reporter.error("Implement applyDefaults for vbox version %s" % (self.fpApiVer,)); + #fRc = oSession.setupPreferredConfig(); + fRc = False; + + # Apply the specified configuration: + if fRc and sDvdImage is not None: + #fRc = oSession.insertDvd(sDvdImage); # attachDvd + reporter.error('Implement: oSession.insertDvd(%s)' % (sDvdImage,)); + fRc = False; + + if fRc and fFastBootLogo is not None: + fRc = oSession.setupBootLogo(fFastBootLogo); + + if fRc and (eNic0AttachType is not None or (sNic0NetName is not None and sNic0NetName != 'default')): + fRc = oSession.setNicAttachment(eNic0AttachType, sNic0NetName, 0); + if fRc and sNic0MacAddr is not None: + if sNic0MacAddr == 'grouped': + sNic0MacAddr = '%02X' % (iGroup,); + fRc = oSession.setNicMacAddress(sNic0MacAddr, 0); + # Needed to reach the host (localhost) from the guest. See xTracker #9896. + if fRc and self.fpApiVer >= 7.0: + fRc = oSession.setNicLocalhostReachable(True, 0); + + if fRc and self.fEnableVrdp: + fRc = oSession.setupVrdp(True, self.uVrdpBasePort + iGroup); + + if fRc and fVmmDevTestingPart is not None: + fRc = oSession.enableVmmDevTestingPart(fVmmDevTestingPart, fVmmDevTestingMmio); + + if fRc and sCom1RawFile: + fRc = oSession.setupSerialToRawFile(0, sCom1RawFile); + + # Save the settings if we were successfull, otherwise discard them. + if fRc: + fRc = oSession.saveSettings(); + if not fRc: + oSession.discardSettings(True); + oSession.close(); + + if fRc is True: + # If we've been successful, add the VM to the list and return it. + # success. + reporter.log('created "%s" with name "%s"' % (oVM.id, sName, )); + self.aoVMs.append(oVM); + self.logVmInfo(oVM); # testing... + return oVM; + + # Failed. Unregister the machine and delete it. + if self.fpApiVer >= 4.0: + try: oVM.unregister(vboxcon.CleanupMode_Full); + except: reporter.logXcpt(); + try: + if self.fpApiVer >= 4.3: + oProgress = oVM.deleteConfig([]); + else: + oProgress = oVM.delete([]); + self.waitOnProgress(oProgress); + except: + reporter.logXcpt(); + else: + try: self.oVBox.unregisterMachine(oVM.id); + except: reporter.logXcpt(); + try: oVM.deleteSettings(); + except: reporter.logXcpt(); + return None; + + def addTestMachine(self, sNameOrId, fQuiet = False): + """ + Adds an already existing (that is, configured) test VM to the + test VM list. + + Returns the VM object on success, None if failed. + """ + # find + add the VM to the list. + oVM = None; + try: + if self.fpApiVer >= 4.0: + oVM = self.oVBox.findMachine(sNameOrId); + else: + reporter.error('fpApiVer=%s - did you remember to initialize the API' % (self.fpApiVer,)); + except: + reporter.errorXcpt('could not find vm "%s"' % (sNameOrId,)); + + if oVM: + self.aoVMs.append(oVM); + if not fQuiet: + reporter.log('Added "%s" with name "%s"' % (oVM.id, sNameOrId)); + self.logVmInfo(oVM); + return oVM; + + def forgetTestMachine(self, oVM, fQuiet = False): + """ + Forget about an already known test VM in the test VM list. + + Returns True on success, False if failed. + """ + try: + sUuid = oVM.id; + sName = oVM.name; + except: + reporter.errorXcpt('failed to get the UUID for VM "%s"' % (oVM,)); + return False; + try: + self.aoVMs.remove(oVM); + if not fQuiet: + reporter.log('Removed "%s" with name "%s"' % (sUuid, sName)); + except: + reporter.errorXcpt('could not find vm "%s"' % (sName,)); + return False; + return True; + + def openSession(self, oVM): + """ + Opens a session for the VM. Returns the a Session wrapper object that + will automatically close the session when the wrapper goes out of scope. + + On failure None is returned and an error is logged. + """ + try: + sUuid = oVM.id; + except: + reporter.errorXcpt('failed to get the UUID for VM "%s"' % (oVM,)); + return None; + + # This loop is a kludge to deal with us racing the closing of the + # direct session of a previous VM run. See waitOnDirectSessionClose. + for i in range(10): + try: + if self.fpApiVer <= 3.2: + oSession = self.oVBoxMgr.openMachineSession(sUuid); + else: + oSession = self.oVBoxMgr.openMachineSession(oVM); + break; + except: + if i == 9: + reporter.errorXcpt('failed to open session for "%s" ("%s")' % (sUuid, oVM)); + return None; + if i > 0: + reporter.logXcpt('warning: failed to open session for "%s" ("%s") - retrying in %u secs' % (sUuid, oVM, i)); + self.waitOnDirectSessionClose(oVM, 5000 + i * 1000); + from testdriver.vboxwrappers import SessionWrapper; + return SessionWrapper(oSession, oVM, self.oVBox, self.oVBoxMgr, self, False); + + # + # Guest locations. + # + + @staticmethod + def getGuestTempDir(oTestVm): + """ + Helper for finding a temporary directory in the test VM. + + Note! It may be necessary to create it! + """ + if oTestVm.isWindows(): + return "C:\\Temp"; + if oTestVm.isOS2(): + return "C:\\Temp"; + return '/var/tmp'; + + @staticmethod + def getGuestSystemDir(oTestVm, sPathPrefix = ''): + """ + Helper for finding a system directory in the test VM that we can play around with. + sPathPrefix can be used to specify other directories, such as /usr/local/bin/ or /usr/bin, for instance. + + On Windows this is always the System32 directory, so this function can be used as + basis for locating other files in or under that directory. + """ + if oTestVm.isWindows(): + return oTestVm.pathJoin(TestDriver.getGuestWinDir(oTestVm), 'System32'); + if oTestVm.isOS2(): + return 'C:\\OS2\\DLL'; + + # OL / RHEL symlinks "/bin"/ to "/usr/bin". To avoid (unexpectedly) following symlinks, use "/usr/bin" then instead. + if not sPathPrefix \ + and oTestVm.sKind in ('Oracle_64', 'Oracle'): ## @todo Does this apply for "RedHat" as well? + return "/usr/bin"; + + return sPathPrefix + "/bin"; + + @staticmethod + def getGuestSystemAdminDir(oTestVm, sPathPrefix = ''): + """ + Helper for finding a system admin directory ("sbin") in the test VM that we can play around with. + sPathPrefix can be used to specify other directories, such as /usr/local/sbin/ or /usr/sbin, for instance. + + On Windows this is always the System32 directory, so this function can be used as + basis for locating other files in or under that directory. + On UNIX-y systems this always is the "sh" shell to guarantee a common shell syntax. + """ + if oTestVm.isWindows(): + return oTestVm.pathJoin(TestDriver.getGuestWinDir(oTestVm), 'System32'); + if oTestVm.isOS2(): + return 'C:\\OS2\\DLL'; ## @todo r=andy Not sure here. + + # OL / RHEL symlinks "/sbin"/ to "/usr/sbin". To avoid (unexpectedly) following symlinks, use "/usr/sbin" then instead. + if not sPathPrefix \ + and oTestVm.sKind in ('Oracle_64', 'Oracle'): ## @todo Does this apply for "RedHat" as well? + return "/usr/sbin"; + + return sPathPrefix + "/sbin"; + + @staticmethod + def getGuestWinDir(oTestVm): + """ + Helper for finding the Windows directory in the test VM that we can play around with. + ASSUMES that we always install Windows on drive C. + + Returns the Windows directory, or an empty string when executed on a non-Windows guest (asserts). + """ + sWinDir = ''; + if oTestVm.isWindows(): + if oTestVm.sKind in ['WindowsNT4', 'WindowsNT3x',]: + sWinDir = 'C:\\WinNT\\'; + else: + sWinDir = 'C:\\Windows\\'; + assert sWinDir != '', 'Retrieving Windows directory for non-Windows OS'; + return sWinDir; + + @staticmethod + def getGuestSystemShell(oTestVm): + """ + Helper for finding the default system shell in the test VM. + """ + if oTestVm.isWindows(): + return TestDriver.getGuestSystemDir(oTestVm) + '\\cmd.exe'; + if oTestVm.isOS2(): + return TestDriver.getGuestSystemDir(oTestVm) + '\\..\\CMD.EXE'; + return "/bin/sh"; + + @staticmethod + def getGuestSystemFileForReading(oTestVm): + """ + Helper for finding a file in the test VM that we can read. + """ + if oTestVm.isWindows(): + return TestDriver.getGuestSystemDir(oTestVm) + '\\ntdll.dll'; + if oTestVm.isOS2(): + return TestDriver.getGuestSystemDir(oTestVm) + '\\DOSCALL1.DLL'; + return "/bin/sh"; + + def getVmByName(self, sName): + """ + Get a test VM by name. Returns None if not found, logged. + """ + # Look it up in our 'cache'. + for oVM in self.aoVMs: + try: + #reporter.log2('cur: %s / %s (oVM=%s)' % (oVM.name, oVM.id, oVM)); + if oVM.name == sName: + return oVM; + except: + reporter.errorXcpt('failed to get the name from the VM "%s"' % (oVM)); + + # Look it up the standard way. + return self.addTestMachine(sName, fQuiet = True); + + def getVmByUuid(self, sUuid): + """ + Get a test VM by uuid. Returns None if not found, logged. + """ + # Look it up in our 'cache'. + for oVM in self.aoVMs: + try: + if oVM.id == sUuid: + return oVM; + except: + reporter.errorXcpt('failed to get the UUID from the VM "%s"' % (oVM)); + + # Look it up the standard way. + return self.addTestMachine(sUuid, fQuiet = True); + + def waitOnProgress(self, oProgress, cMsTimeout = 1000000, fErrorOnTimeout = True, cMsInterval = 1000): + """ + Waits for a progress object to complete. Returns the status code. + """ + # Wait for progress no longer than cMsTimeout time period. + tsStart = datetime.datetime.now() + while True: + self.processPendingEvents(); + try: + if oProgress.completed: + break; + except: + return -1; + self.processPendingEvents(); + + tsNow = datetime.datetime.now() + tsDelta = tsNow - tsStart + if ((tsDelta.microseconds + tsDelta.seconds * 1000000) // 1000) > cMsTimeout: + if fErrorOnTimeout: + reporter.errorTimeout('Timeout while waiting for progress.') + return -1 + + reporter.doPollWork('vbox.TestDriver.waitOnProgress'); + try: oProgress.waitForCompletion(cMsInterval); + except: return -2; + + try: rc = oProgress.resultCode; + except: rc = -2; + self.processPendingEvents(); + return rc; + + def waitOnDirectSessionClose(self, oVM, cMsTimeout): + """ + Waits for the VM process to close it's current direct session. + + Returns None. + """ + # Get the original values so we're not subject to + try: + eCurState = oVM.sessionState; + if self.fpApiVer >= 5.0: + sCurName = sOrgName = oVM.sessionName; + else: + sCurName = sOrgName = oVM.sessionType; + if self.fpApiVer >= 4.2: + iCurPid = iOrgPid = oVM.sessionPID; + else: + iCurPid = iOrgPid = oVM.sessionPid; + except Exception as oXcpt: + if ComError.notEqual(oXcpt, ComError.E_ACCESSDENIED): + reporter.logXcpt(); + self.processPendingEvents(); + return None; + self.processPendingEvents(); + + msStart = base.timestampMilli(); + while iCurPid == iOrgPid \ + and sCurName == sOrgName \ + and sCurName != '' \ + and base.timestampMilli() - msStart < cMsTimeout \ + and eCurState in (vboxcon.SessionState_Unlocking, vboxcon.SessionState_Spawning, vboxcon.SessionState_Locked,): + self.processEvents(1000); + try: + eCurState = oVM.sessionState; + sCurName = oVM.sessionName if self.fpApiVer >= 5.0 else oVM.sessionType; + iCurPid = oVM.sessionPID if self.fpApiVer >= 4.2 else oVM.sessionPid; + except Exception as oXcpt: + if ComError.notEqual(oXcpt, ComError.E_ACCESSDENIED): + reporter.logXcpt(); + break; + self.processPendingEvents(); + self.processPendingEvents(); + return None; + + def uploadStartupLogFile(self, oVM, sVmName): + """ + Uploads the VBoxStartup.log when present. + """ + fRc = True; + try: + sLogFile = os.path.join(oVM.logFolder, 'VBoxHardening.log'); + except: + reporter.logXcpt(); + fRc = False; + else: + if os.path.isfile(sLogFile): + reporter.addLogFile(sLogFile, 'log/release/vm', '%s hardening log' % (sVmName, ), + sAltName = '%s-%s' % (sVmName, os.path.basename(sLogFile),)); + return fRc; + + def annotateAndUploadProcessReport(self, sProcessReport, sFilename, sKind, sDesc): + """ + Annotates the given VM process report and uploads it if successfull. + """ + fRc = False; + if self.oBuild is not None and self.oBuild.sInstallPath is not None: + oResolver = btresolver.BacktraceResolver(self.sScratchPath, self.oBuild.sInstallPath, + self.getBuildOs(), self.getBuildArch(), + fnLog = reporter.log); + fRcTmp = oResolver.prepareEnv(); + if fRcTmp: + reporter.log('Successfully prepared environment'); + sReportDbgSym = oResolver.annotateReport(sProcessReport); + if sReportDbgSym and len(sReportDbgSym) > 8: + reporter.addLogString(sReportDbgSym, sFilename, sKind, sDesc); + fRc = True; + else: + reporter.log('Annotating report failed'); + oResolver.cleanupEnv(); + return fRc; + + def startVmEx(self, oVM, fWait = True, sType = None, sName = None, asEnv = None): # pylint: disable=too-many-locals,too-many-statements + """ + Start the VM, returning the VM session and progress object on success. + The session is also added to the task list and to the aoRemoteSessions set. + + asEnv is a list of string on the putenv() form. + + On failure (None, None) is returned and an error is logged. + """ + # Massage and check the input. + if sType is None: + sType = self.sSessionType; + if sName is None: + try: sName = oVM.name; + except: sName = 'bad-vm-handle'; + reporter.log('startVmEx: sName=%s fWait=%s sType=%s' % (sName, fWait, sType)); + if oVM is None: + return (None, None); + + ## @todo Do this elsewhere. + # Hack alert. Disables all annoying GUI popups. + if sType == 'gui' and not self.aoRemoteSessions: + try: + self.oVBox.setExtraData('GUI/Input/AutoCapture', 'false'); + if self.fpApiVer >= 3.2: + self.oVBox.setExtraData('GUI/LicenseAgreed', '8'); + else: + self.oVBox.setExtraData('GUI/LicenseAgreed', '7'); + self.oVBox.setExtraData('GUI/RegistrationData', 'triesLeft=0'); + self.oVBox.setExtraData('GUI/SUNOnlineData', 'triesLeft=0'); + self.oVBox.setExtraData('GUI/SuppressMessages', 'confirmVMReset,remindAboutMouseIntegrationOn,' + 'remindAboutMouseIntegrationOff,remindAboutPausedVMInput,confirmInputCapture,' + 'confirmGoingFullscreen,remindAboutInaccessibleMedia,remindAboutWrongColorDepth,' + 'confirmRemoveMedium,allPopupPanes,allMessageBoxes,all'); + self.oVBox.setExtraData('GUI/UpdateDate', 'never'); + self.oVBox.setExtraData('GUI/PreventBetaWarning', self.oVBox.version); + except: + reporter.logXcpt(); + + # The UUID for the name. + try: + sUuid = oVM.id; + except: + reporter.errorXcpt('failed to get the UUID for VM "%s"' % (oVM)); + return (None, None); + self.processPendingEvents(); + + # Construct the environment. + self.sSessionLogFile = '%s/VM-%s.log' % (self.sScratchPath, sUuid); + try: os.remove(self.sSessionLogFile); + except: pass; + if self.sLogSessionDest: + sLogDest = self.sLogSessionDest; + else: + sLogDest = 'file=%s' % (self.sSessionLogFile,); + asEnvFinal = [ + 'VBOX_LOG=%s' % (self.sLogSessionGroups,), + 'VBOX_LOG_FLAGS=%s' % (self.sLogSessionFlags,), + 'VBOX_LOG_DEST=nodeny %s' % (sLogDest,), + 'VBOX_RELEASE_LOG_FLAGS=append time', + ]; + if sType == 'gui': + asEnvFinal.append('VBOX_GUI_DBG_ENABLED=1'); + if asEnv is not None and asEnv: + asEnvFinal += asEnv; + + reporter.log2('Session environment:\n%s' % (asEnvFinal,)); + + # Shortcuts for local testing. + oProgress = oWrapped = None; + oTestVM = self.oTestVmSet.findTestVmByName(sName) if self.oTestVmSet is not None else None; + try: + if oTestVM is not None \ + and oTestVM.fSnapshotRestoreCurrent is True: + if oVM.state is vboxcon.MachineState_Running: + reporter.log2('Machine "%s" already running.' % (sName,)); + oProgress = None; + oWrapped = self.openSession(oVM); + else: + reporter.log2('Checking if snapshot for machine "%s" exists.' % (sName,)); + oSessionWrapperRestore = self.openSession(oVM); + if oSessionWrapperRestore is not None: + oSnapshotCur = oVM.currentSnapshot; + if oSnapshotCur is not None: + reporter.log2('Restoring snapshot for machine "%s".' % (sName,)); + oSessionWrapperRestore.restoreSnapshot(oSnapshotCur); + reporter.log2('Current snapshot for machine "%s" restored.' % (sName,)); + else: + reporter.log('warning: no current snapshot for machine "%s" found.' % (sName,)); + oSessionWrapperRestore.close(); + except: + reporter.errorXcpt(); + return (None, None); + + oSession = None; # Must be initialized, otherwise the log statement at the end of the function can fail. + + # Open a remote session, wait for this operation to complete. + # (The loop is a kludge to deal with us racing the closing of the + # direct session of a previous VM run. See waitOnDirectSessionClose.) + if oWrapped is None: + for i in range(10): + try: + if self.fpApiVer < 4.3 \ + or (self.fpApiVer == 4.3 and not hasattr(self.oVBoxMgr, 'getSessionObject')): + oSession = self.oVBoxMgr.mgr.getSessionObject(self.oVBox); # pylint: disable=no-member + elif self.fpApiVer < 5.2 \ + or (self.fpApiVer == 5.2 and hasattr(self.oVBoxMgr, 'vbox')): + oSession = self.oVBoxMgr.getSessionObject(self.oVBox); # pylint: disable=no-member + else: + oSession = self.oVBoxMgr.getSessionObject(); # pylint: disable=no-member,no-value-for-parameter + if self.fpApiVer < 3.3: + oProgress = self.oVBox.openRemoteSession(oSession, sUuid, sType, '\n'.join(asEnvFinal)); + else: + if self.uApiRevision >= self.makeApiRevision(6, 1, 0, 1): + oProgress = oVM.launchVMProcess(oSession, sType, asEnvFinal); + else: + oProgress = oVM.launchVMProcess(oSession, sType, '\n'.join(asEnvFinal)); + break; + except: + if i == 9: + reporter.errorXcpt('failed to start VM "%s" ("%s"), aborting.' % (sUuid, sName)); + return (None, None); + oSession = None; + if i >= 0: + reporter.logXcpt('warning: failed to start VM "%s" ("%s") - retrying in %u secs.' % (sUuid, oVM, i)); # pylint: disable=line-too-long + self.waitOnDirectSessionClose(oVM, 5000 + i * 1000); + if fWait and oProgress is not None: + rc = self.waitOnProgress(oProgress); + if rc < 0: + self.waitOnDirectSessionClose(oVM, 5000); + + # VM failed to power up, still collect VBox.log, need to wrap the session object + # in order to use the helper for adding the log files to the report. + from testdriver.vboxwrappers import SessionWrapper; + oTmp = SessionWrapper(oSession, oVM, self.oVBox, self.oVBoxMgr, self, True, sName, self.sSessionLogFile); + oTmp.addLogsToReport(); + + # Try to collect a stack trace of the process for further investigation of any startup hangs. + uPid = oTmp.getPid(); + if uPid is not None: + sHostProcessInfoHung = utils.processGetInfo(uPid, fSudo = True); + if sHostProcessInfoHung is not None: + reporter.log('Trying to annotate the hung VM startup process report, please stand by...'); + fRcTmp = self.annotateAndUploadProcessReport(sHostProcessInfoHung, 'vmprocess-startup-hung.log', + 'process/report/vm', 'Annotated hung VM process state during startup'); # pylint: disable=line-too-long + # Upload the raw log for manual annotation in case resolving failed. + if not fRcTmp: + reporter.log('Failed to annotate hung VM process report, uploading raw report'); + reporter.addLogString(sHostProcessInfoHung, 'vmprocess-startup-hung.log', 'process/report/vm', + 'Hung VM process state during startup'); + + try: + if oSession is not None: + oSession.close(); + except: pass; + reportError(oProgress, 'failed to open session for "%s"' % (sName)); + self.uploadStartupLogFile(oVM, sName); + return (None, None); + reporter.log2('waitOnProgress -> %s' % (rc,)); + + # Wrap up the session object and push on to the list before returning it. + if oWrapped is None: + from testdriver.vboxwrappers import SessionWrapper; + oWrapped = SessionWrapper(oSession, oVM, self.oVBox, self.oVBoxMgr, self, True, sName, self.sSessionLogFile); + + oWrapped.registerEventHandlerForTask(); + self.aoRemoteSessions.append(oWrapped); + if oWrapped is not self.aoRemoteSessions[len(self.aoRemoteSessions) - 1]: + reporter.error('not by reference: oWrapped=%s aoRemoteSessions[%s]=%s' + % (oWrapped, len(self.aoRemoteSessions) - 1, + self.aoRemoteSessions[len(self.aoRemoteSessions) - 1])); + self.addTask(oWrapped); + + reporter.log2('startVmEx: oSession=%s, oSessionWrapper=%s, oProgress=%s' % (oSession, oWrapped, oProgress)); + + from testdriver.vboxwrappers import ProgressWrapper; + return (oWrapped, ProgressWrapper(oProgress, self.oVBoxMgr, self, + 'starting %s' % (sName,)) if oProgress else None); + + def startVm(self, oVM, sType=None, sName = None, asEnv = None): + """ Simplified version of startVmEx. """ + oSession, _ = self.startVmEx(oVM, True, sType, sName, asEnv = asEnv); + return oSession; + + def startVmByNameEx(self, sName, fWait=True, sType=None, asEnv = None): + """ + Start the VM, returning the VM session and progress object on success. + The session is also added to the task list and to the aoRemoteSessions set. + + On failure (None, None) is returned and an error is logged. + """ + oVM = self.getVmByName(sName); + if oVM is None: + return (None, None); + return self.startVmEx(oVM, fWait, sType, sName, asEnv = asEnv); + + def startVmByName(self, sName, sType=None, asEnv = None): + """ + Start the VM, returning the VM session on success. The session is + also added to the task list and to the aoRemoteSessions set. + + On failure None is returned and an error is logged. + """ + oSession, _ = self.startVmByNameEx(sName, True, sType, asEnv = asEnv); + return oSession; + + def terminateVmBySession(self, oSession, oProgress = None, fTakeScreenshot = None): # pylint: disable=too-many-statements + """ + Terminates the VM specified by oSession and adds the release logs to + the test report. + + This will try achieve this by using powerOff, but will resort to + tougher methods if that fails. + + The session will always be removed from the task list. + The session will be closed unless we fail to kill the process. + The session will be removed from the remote session list if closed. + + The progress object (a wrapper!) is for teleportation and similar VM + operations, it will be attempted canceled before powering off the VM. + Failures are logged but ignored. + The progress object will always be removed from the task list. + + Returns True if powerOff and session close both succeed. + Returns False if on failure (logged), including when we successfully + kill the VM process. + """ + reporter.log2('terminateVmBySession: oSession=%s (pid=%s) oProgress=%s' % (oSession.sName, oSession.getPid(), oProgress)); + + # Call getPid first to make sure the PID is cached in the wrapper. + oSession.getPid(); + + # + # If the host is out of memory, just skip all the info collection as it + # requires memory too and seems to wedge. + # + sHostProcessInfo = None; + sHostProcessInfoHung = None; + sLastScreenshotPath = None; + sOsKernelLog = None; + sVgaText = None; + asMiscInfos = []; + + if not oSession.fHostMemoryLow: + # Try to fetch the VM process info before meddling with its state. + if self.fAlwaysUploadLogs or reporter.testErrorCount() > 0: + sHostProcessInfo = utils.processGetInfo(oSession.getPid(), fSudo = True); + + # + # Pause the VM if we're going to take any screenshots or dig into the + # guest. Failures are quitely ignored. + # + if self.fAlwaysUploadLogs or reporter.testErrorCount() > 0: + try: + if oSession.oVM.state in [ vboxcon.MachineState_Running, + vboxcon.MachineState_LiveSnapshotting, + vboxcon.MachineState_Teleporting ]: + oSession.o.console.pause(); + except: + reporter.logXcpt(); + + # + # Take Screenshot and upload it (see below) to Test Manager if appropriate/requested. + # + if fTakeScreenshot is True or self.fAlwaysUploadScreenshots or reporter.testErrorCount() > 0: + sLastScreenshotPath = os.path.join(self.sScratchPath, "LastScreenshot-%s.png" % oSession.sName); + fRc = oSession.takeScreenshot(sLastScreenshotPath); + if fRc is not True: + sLastScreenshotPath = None; + + # Query the OS kernel log from the debugger if appropriate/requested. + if self.fAlwaysUploadLogs or reporter.testErrorCount() > 0: + sOsKernelLog = oSession.queryOsKernelLog(); + + # Do "info vgatext all" separately. + if self.fAlwaysUploadLogs or reporter.testErrorCount() > 0: + sVgaText = oSession.queryDbgInfoVgaText(); + + # Various infos (do after kernel because of symbols). + if self.fAlwaysUploadLogs or reporter.testErrorCount() > 0: + # Dump the guest stack for all CPUs. + cCpus = oSession.getCpuCount(); + if cCpus > 0: + for iCpu in xrange(0, cCpus): + sThis = oSession.queryDbgGuestStack(iCpu); + if sThis: + asMiscInfos += [ + '================ start guest stack VCPU %s ================\n' % (iCpu,), + sThis, + '================ end guest stack VCPU %s ==================\n' % (iCpu,), + ]; + + for sInfo, sArg in [ ('mode', 'all'), + ('fflags', ''), + ('cpumguest', 'verbose all'), + ('cpumguestinstr', 'symbol all'), + ('exits', ''), + ('pic', ''), + ('apic', ''), + ('apiclvt', ''), + ('apictimer', ''), + ('ioapic', ''), + ('pit', ''), + ('phys', ''), + ('clocks', ''), + ('timers', ''), + ('gdt', ''), + ('ldt', ''), + ]: + if sInfo in ['apic',] and self.fpApiVer < 5.1: # asserts and burns + continue; + sThis = oSession.queryDbgInfo(sInfo, sArg); + if sThis: + if sThis[-1] != '\n': + sThis += '\n'; + asMiscInfos += [ + '================ start %s %s ================\n' % (sInfo, sArg), + sThis, + '================ end %s %s ==================\n' % (sInfo, sArg), + ]; + + # + # Terminate the VM + # + + # Cancel the progress object if specified. + if oProgress is not None: + if not oProgress.isCompleted() and oProgress.isCancelable(): + reporter.log2('terminateVmBySession: canceling "%s"...' % (oProgress.sName)); + try: + oProgress.o.cancel(); + except: + reporter.logXcpt(); + else: + oProgress.wait(); + self.removeTask(oProgress); + + # Check if the VM has terminated by itself before powering it off. + fClose = True; + fRc = True; + if oSession.needsPoweringOff(): + reporter.log('terminateVmBySession: powering off "%s"...' % (oSession.sName,)); + fRc = oSession.powerOff(fFudgeOnFailure = False); + if fRc is not True: + # power off failed, try terminate it in a nice manner. + fRc = False; + uPid = oSession.getPid(); + if uPid is not None: + # + # Collect some information about the VM process first to have + # some state information for further investigation why powering off failed. + # + sHostProcessInfoHung = utils.processGetInfo(uPid, fSudo = True); + + # Exterminate... + reporter.error('terminateVmBySession: Terminating PID %u (VM %s)' % (uPid, oSession.sName)); + fClose = base.processTerminate(uPid); + if fClose is True: + self.waitOnDirectSessionClose(oSession.oVM, 5000); + fClose = oSession.waitForTask(1000); + + if fClose is not True: + # Being nice failed... + reporter.error('terminateVmBySession: Termination failed, trying to kill PID %u (VM %s) instead' \ + % (uPid, oSession.sName)); + fClose = base.processKill(uPid); + if fClose is True: + self.waitOnDirectSessionClose(oSession.oVM, 5000); + fClose = oSession.waitForTask(1000); + if fClose is not True: + reporter.error('terminateVmBySession: Failed to kill PID %u (VM %s)' % (uPid, oSession.sName)); + + # The final steps. + if fClose is True: + reporter.log('terminateVmBySession: closing session "%s"...' % (oSession.sName,)); + oSession.close(); + self.waitOnDirectSessionClose(oSession.oVM, 10000); + try: + eState = oSession.oVM.state; + except: + reporter.logXcpt(); + else: + if eState == vboxcon.MachineState_Aborted: + reporter.error('terminateVmBySession: The VM "%s" aborted!' % (oSession.sName,)); + self.removeTask(oSession); + + # + # Add the release log, debug log and a screenshot of the VM to the test report. + # + if self.fAlwaysUploadLogs or reporter.testErrorCount() > 0: + oSession.addLogsToReport(); + + # Add a screenshot if it has been requested and taken successfully. + if sLastScreenshotPath is not None: + if reporter.testErrorCount() > 0: + reporter.addLogFile(sLastScreenshotPath, 'screenshot/failure', 'Last VM screenshot'); + else: + reporter.addLogFile(sLastScreenshotPath, 'screenshot/success', 'Last VM screenshot'); + + # Add the guest OS log if it has been requested and taken successfully. + if sOsKernelLog is not None: + reporter.addLogString(sOsKernelLog, 'kernel.log', 'log/guest/kernel', 'Guest OS kernel log'); + + # Add "info vgatext all" if we've got it. + if sVgaText is not None: + reporter.addLogString(sVgaText, 'vgatext.txt', 'info/vgatext', 'info vgatext all'); + + # Add the "info xxxx" items if we've got any. + if asMiscInfos: + reporter.addLogString(u''.join(asMiscInfos), 'info.txt', 'info/collection', 'A bunch of info items.'); + + # Add the host process info if we were able to retrieve it. + if sHostProcessInfo is not None: + reporter.log('Trying to annotate the VM process report, please stand by...'); + fRcTmp = self.annotateAndUploadProcessReport(sHostProcessInfo, 'vmprocess.log', + 'process/report/vm', 'Annotated VM process state'); + # Upload the raw log for manual annotation in case resolving failed. + if not fRcTmp: + reporter.log('Failed to annotate VM process report, uploading raw report'); + reporter.addLogString(sHostProcessInfo, 'vmprocess.log', 'process/report/vm', 'VM process state'); + + # Add the host process info for failed power off attempts if we were able to retrieve it. + if sHostProcessInfoHung is not None: + reporter.log('Trying to annotate the hung VM process report, please stand by...'); + fRcTmp = self.annotateAndUploadProcessReport(sHostProcessInfoHung, 'vmprocess-hung.log', + 'process/report/vm', 'Annotated hung VM process state'); + # Upload the raw log for manual annotation in case resolving failed. + if not fRcTmp: + reporter.log('Failed to annotate hung VM process report, uploading raw report'); + fRcTmp = reporter.addLogString(sHostProcessInfoHung, 'vmprocess-hung.log', 'process/report/vm', + 'Hung VM process state'); + if not fRcTmp: + try: reporter.log('******* START vmprocess-hung.log *******\n%s\n******* END vmprocess-hung.log *******\n' + % (sHostProcessInfoHung,)); + except: pass; # paranoia + + # Upload the screen video recordings if appropriate. + if self.fAlwaysUploadRecordings or reporter.testErrorCount() > 0: + reporter.log2('Uploading %d screen recordings ...' % (len(self.adRecordingFiles),)); + for dRecFile in self.adRecordingFiles: + reporter.log2('Uploading screen recording "%s" (screen %d)' % (dRecFile['file'], dRecFile['id'])); + reporter.addLogFile(dRecFile['file'], + 'screenrecording/failure' if reporter.testErrorCount() > 0 else 'screenrecording/success', + 'Recording of screen #%d' % (dRecFile['id'],)); + + return fRc; + + + # + # Some information query functions (mix). + # + # Methods require the VBox API. If the information is provided by both + # the testboxscript as well as VBox API, we'll check if it matches. + # + + def _hasHostCpuFeature(self, sEnvVar, sEnum, fpApiMinVer, fQuiet): + """ + Common Worker for hasHostNestedPaging() and hasHostHwVirt(). + + Returns True / False. + Raises exception on environment / host mismatch. + """ + fEnv = os.environ.get(sEnvVar, None); + if fEnv is not None: + fEnv = fEnv.lower() not in [ 'false', 'f', 'not', 'no', 'n', '0', ]; + + fVBox = None; + self.importVBoxApi(); + if self.fpApiVer >= fpApiMinVer and hasattr(vboxcon, sEnum): + try: + fVBox = self.oVBox.host.getProcessorFeature(getattr(vboxcon, sEnum)); + except: + if not fQuiet: + reporter.logXcpt(); + + if fVBox is not None: + if fEnv is not None: + if fEnv != fVBox and not fQuiet: + reporter.log('TestBox configuration overwritten: fVBox=%s (%s) vs. fEnv=%s (%s)' + % (fVBox, sEnum, fEnv, sEnvVar)); + return fEnv; + return fVBox; + if fEnv is not None: + return fEnv; + return False; + + def hasHostHwVirt(self, fQuiet = False): + """ + Checks if hardware assisted virtualization is supported by the host. + + Returns True / False. + Raises exception on environment / host mismatch. + """ + return self._hasHostCpuFeature('TESTBOX_HAS_HW_VIRT', 'ProcessorFeature_HWVirtEx', 3.1, fQuiet); + + def hasHostNestedPaging(self, fQuiet = False): + """ + Checks if nested paging is supported by the host. + + Returns True / False. + Raises exception on environment / host mismatch. + """ + return self._hasHostCpuFeature('TESTBOX_HAS_NESTED_PAGING', 'ProcessorFeature_NestedPaging', 4.2, fQuiet) \ + and self.hasHostHwVirt(fQuiet); + + def hasHostNestedHwVirt(self, fQuiet = False): + """ + Checks if nested hardware-assisted virtualization is supported by the host. + + Returns True / False. + Raises exception on environment / host mismatch. + """ + return self._hasHostCpuFeature('TESTBOX_HAS_NESTED_HWVIRT', 'ProcessorFeature_NestedHWVirt', 6.0, fQuiet) \ + and self.hasHostHwVirt(fQuiet); + + def hasHostLongMode(self, fQuiet = False): + """ + Checks if the host supports 64-bit guests. + + Returns True / False. + Raises exception on environment / host mismatch. + """ + # Note that the testboxscript doesn't export this variable atm. + return self._hasHostCpuFeature('TESTBOX_HAS_LONG_MODE', 'ProcessorFeature_LongMode', 3.1, fQuiet); + + def getHostCpuCount(self, fQuiet = False): + """ + Returns the number of CPUs on the host. + + Returns True / False. + Raises exception on environment / host mismatch. + """ + cEnv = os.environ.get('TESTBOX_CPU_COUNT', None); + if cEnv is not None: + cEnv = int(cEnv); + + try: + cVBox = self.oVBox.host.processorOnlineCount; + except: + if not fQuiet: + reporter.logXcpt(); + cVBox = None; + + if cVBox is not None: + if cEnv is not None: + assert cVBox == cEnv, 'Misconfigured TestBox: VBox: %u CPUs, testboxscript: %u CPUs' % (cVBox, cEnv); + return cVBox; + if cEnv is not None: + return cEnv; + return 1; + + def _getHostCpuDesc(self, fQuiet = False): + """ + Internal method used for getting the host CPU description from VBoxSVC. + Returns description string, on failure an empty string is returned. + """ + try: + return self.oVBox.host.getProcessorDescription(0); + except: + if not fQuiet: + reporter.logXcpt(); + return ''; + + def isHostCpuAmd(self, fQuiet = False): + """ + Checks if the host CPU vendor is AMD. + + Returns True / False. + """ + sCpuDesc = self._getHostCpuDesc(fQuiet); + return 'AMD' in sCpuDesc or sCpuDesc == 'AuthenticAMD'; + + def isHostCpuIntel(self, fQuiet = False): + """ + Checks if the host CPU vendor is Intel. + + Returns True / False. + """ + sCpuDesc = self._getHostCpuDesc(fQuiet); + return sCpuDesc.startswith("Intel") or sCpuDesc == 'GenuineIntel'; + + def isHostCpuVia(self, fQuiet = False): + """ + Checks if the host CPU vendor is VIA (or Centaur). + + Returns True / False. + """ + sCpuDesc = self._getHostCpuDesc(fQuiet); + return sCpuDesc.startswith("VIA") or sCpuDesc == 'CentaurHauls'; + + def isHostCpuShanghai(self, fQuiet = False): + """ + Checks if the host CPU vendor is Shanghai (or Zhaoxin). + + Returns True / False. + """ + sCpuDesc = self._getHostCpuDesc(fQuiet); + return sCpuDesc.startswith("ZHAOXIN") or sCpuDesc.strip(' ') == 'Shanghai'; + + def isHostCpuP4(self, fQuiet = False): + """ + Checks if the host CPU is a Pentium 4 / Pentium D. + + Returns True / False. + """ + if not self.isHostCpuIntel(fQuiet): + return False; + + (uFamilyModel, _, _, _) = self.oVBox.host.getProcessorCPUIDLeaf(0, 0x1, 0); + return ((uFamilyModel >> 8) & 0xf) == 0xf; + + def hasRawModeSupport(self, fQuiet = False): + """ + Checks if raw-mode is supported by VirtualBox that the testbox is + configured for it. + + Returns True / False. + Raises no exceptions. + + Note! Differs from the rest in that we don't require the + TESTBOX_WITH_RAW_MODE value to match the API. It is + sometimes helpful to disable raw-mode on individual + test boxes. (This probably goes for + """ + # The environment variable can be used to disable raw-mode. + fEnv = os.environ.get('TESTBOX_WITH_RAW_MODE', None); + if fEnv is not None: + fEnv = fEnv.lower() not in [ 'false', 'f', 'not', 'no', 'n', '0', ]; + if fEnv is False: + return False; + + # Starting with 5.0 GA / RC2 the API can tell us whether VBox was built + # with raw-mode support or not. + self.importVBoxApi(); + if self.fpApiVer >= 5.0: + try: + fVBox = self.oVBox.systemProperties.rawModeSupported; + except: + if not fQuiet: + reporter.logXcpt(); + fVBox = True; + if fVBox is False: + return False; + + return True; + + # + # Testdriver execution methods. + # + + def handleTask(self, oTask, sMethod): + """ + Callback method for handling unknown tasks in the various run loops. + + The testdriver should override this if it already tasks running when + calling startVmAndConnectToTxsViaTcp, txsRunTest or similar methods. + Call super to handle unknown tasks. + + Returns True if handled, False if not. + """ + reporter.error('%s: unknown task %s' % (sMethod, oTask)); + return False; + + def txsDoTask(self, oSession, oTxsSession, fnAsync, aArgs): + """ + Generic TXS task wrapper which waits both on the TXS and the session tasks. + + Returns False on error, logged. + Returns task result on success. + """ + # All async methods ends with the following two args. + cMsTimeout = aArgs[-2]; + fIgnoreErrors = aArgs[-1]; + + fRemoveVm = self.addTask(oSession); + fRemoveTxs = self.addTask(oTxsSession); + + rc = fnAsync(*aArgs); # pylint: disable=star-args + if rc is True: + rc = False; + oTask = self.waitForTasks(cMsTimeout + 1); + if oTask is oTxsSession: + if oTxsSession.isSuccess(): + rc = oTxsSession.getResult(); + elif fIgnoreErrors is True: + reporter.log( 'txsDoTask: task failed (%s)' % (oTxsSession.getLastReply()[1],)); + else: + reporter.error('txsDoTask: task failed (%s)' % (oTxsSession.getLastReply()[1],)); + else: + oTxsSession.cancelTask(); + if oTask is None: + if fIgnoreErrors is True: + reporter.log( 'txsDoTask: The task timed out.'); + else: + reporter.errorTimeout('txsDoTask: The task timed out.'); + elif oTask is oSession: + reporter.error('txsDoTask: The VM terminated unexpectedly'); + else: + if fIgnoreErrors is True: + reporter.log( 'txsDoTask: An unknown task %s was returned' % (oTask,)); + else: + reporter.error('txsDoTask: An unknown task %s was returned' % (oTask,)); + else: + reporter.error('txsDoTask: fnAsync returned %s' % (rc,)); + + if fRemoveTxs: + self.removeTask(oTxsSession); + if fRemoveVm: + self.removeTask(oSession); + return rc; + + # pylint: disable=missing-docstring + + def txsDisconnect(self, oSession, oTxsSession, cMsTimeout = 30000, fIgnoreErrors = False): + return self.txsDoTask(oSession, oTxsSession, oTxsSession.asyncDisconnect, + (self.adjustTimeoutMs(cMsTimeout), fIgnoreErrors)); + + def txsVer(self, oSession, oTxsSession, cMsTimeout = 30000, fIgnoreErrors = False): + return self.txsDoTask(oSession, oTxsSession, oTxsSession.asyncVer, + (self.adjustTimeoutMs(cMsTimeout), fIgnoreErrors)); + + def txsUuid(self, oSession, oTxsSession, cMsTimeout = 30000, fIgnoreErrors = False): + return self.txsDoTask(oSession, oTxsSession, oTxsSession.asyncUuid, + (self.adjustTimeoutMs(cMsTimeout), fIgnoreErrors)); + + def txsMkDir(self, oSession, oTxsSession, sRemoteDir, fMode = 0o700, cMsTimeout = 30000, fIgnoreErrors = False): + return self.txsDoTask(oSession, oTxsSession, oTxsSession.asyncMkDir, + (sRemoteDir, fMode, self.adjustTimeoutMs(cMsTimeout), fIgnoreErrors)); + + def txsMkDirPath(self, oSession, oTxsSession, sRemoteDir, fMode = 0o700, cMsTimeout = 30000, fIgnoreErrors = False): + return self.txsDoTask(oSession, oTxsSession, oTxsSession.asyncMkDirPath, + (sRemoteDir, fMode, self.adjustTimeoutMs(cMsTimeout), fIgnoreErrors)); + + def txsMkSymlink(self, oSession, oTxsSession, sLinkTarget, sLink, cMsTimeout = 30000, fIgnoreErrors = False): + return self.txsDoTask(oSession, oTxsSession, oTxsSession.asyncMkSymlink, + (sLinkTarget, sLink, self.adjustTimeoutMs(cMsTimeout), fIgnoreErrors)); + + def txsRmDir(self, oSession, oTxsSession, sRemoteDir, cMsTimeout = 30000, fIgnoreErrors = False): + return self.txsDoTask(oSession, oTxsSession, oTxsSession.asyncRmDir, + (sRemoteDir, self.adjustTimeoutMs(cMsTimeout), fIgnoreErrors)); + + def txsRmFile(self, oSession, oTxsSession, sRemoteFile, cMsTimeout = 30000, fIgnoreErrors = False): + return self.txsDoTask(oSession, oTxsSession, oTxsSession.asyncRmFile, + (sRemoteFile, self.adjustTimeoutMs(cMsTimeout), fIgnoreErrors)); + + def txsRmSymlink(self, oSession, oTxsSession, sRemoteSymlink, cMsTimeout = 30000, fIgnoreErrors = False): + return self.txsDoTask(oSession, oTxsSession, oTxsSession.asyncRmSymlink, + (sRemoteSymlink, self.adjustTimeoutMs(cMsTimeout), fIgnoreErrors)); + + def txsRmTree(self, oSession, oTxsSession, sRemoteTree, cMsTimeout = 30000, fIgnoreErrors = False): + return self.txsDoTask(oSession, oTxsSession, oTxsSession.asyncRmTree, + (sRemoteTree, self.adjustTimeoutMs(cMsTimeout), fIgnoreErrors)); + + def txsIsDir(self, oSession, oTxsSession, sRemoteDir, cMsTimeout = 30000, fIgnoreErrors = False): + return self.txsDoTask(oSession, oTxsSession, oTxsSession.asyncIsDir, + (sRemoteDir, self.adjustTimeoutMs(cMsTimeout), fIgnoreErrors)); + + def txsIsFile(self, oSession, oTxsSession, sRemoteFile, cMsTimeout = 30000, fIgnoreErrors = False): + return self.txsDoTask(oSession, oTxsSession, oTxsSession.asyncIsFile, + (sRemoteFile, self.adjustTimeoutMs(cMsTimeout), fIgnoreErrors)); + + def txsIsSymlink(self, oSession, oTxsSession, sRemoteSymlink, cMsTimeout = 30000, fIgnoreErrors = False): + return self.txsDoTask(oSession, oTxsSession, oTxsSession.asyncIsSymlink, + (sRemoteSymlink, self.adjustTimeoutMs(cMsTimeout), fIgnoreErrors)); + + def txsCopyFile(self, oSession, oTxsSession, sSrcFile, sDstFile, fMode = 0, cMsTimeout = 30000, fIgnoreErrors = False): + return self.txsDoTask(oSession, oTxsSession, oTxsSession.asyncCopyFile, \ + (sSrcFile, sDstFile, fMode, self.adjustTimeoutMs(cMsTimeout), fIgnoreErrors)); + + def txsUploadFile(self, oSession, oTxsSession, sLocalFile, sRemoteFile, cMsTimeout = 30000, fIgnoreErrors = False): + return self.txsDoTask(oSession, oTxsSession, oTxsSession.asyncUploadFile, \ + (sLocalFile, sRemoteFile, self.adjustTimeoutMs(cMsTimeout), fIgnoreErrors)); + + def txsUploadString(self, oSession, oTxsSession, sContent, sRemoteFile, cMsTimeout = 30000, fIgnoreErrors = False): + return self.txsDoTask(oSession, oTxsSession, oTxsSession.asyncUploadString, \ + (sContent, sRemoteFile, self.adjustTimeoutMs(cMsTimeout), fIgnoreErrors)); + + def txsDownloadFile(self, oSession, oTxsSession, sRemoteFile, sLocalFile, cMsTimeout = 30000, fIgnoreErrors = False): + return self.txsDoTask(oSession, oTxsSession, oTxsSession.asyncDownloadFile, \ + (sRemoteFile, sLocalFile, self.adjustTimeoutMs(cMsTimeout), fIgnoreErrors)); + + def txsDownloadFiles(self, oSession, oTxsSession, aasFiles, fAddToLog = True, fIgnoreErrors = False): + """ + Convenience function to get files from the guest, storing them in the + scratch and adding them to the test result set (optional, but default). + + The aasFiles parameter contains an array of with guest-path + host-path + pairs, optionally a file 'kind', description and an alternative upload + filename can also be specified. + + Host paths are relative to the scratch directory or they must be given + in absolute form. The guest path should be using guest path style. + + Returns True on success. + Returns False on failure (unless fIgnoreErrors is set), logged. + """ + for asEntry in aasFiles: + # Unpack: + sGstFile = asEntry[0]; + sHstFile = asEntry[1]; + sKind = asEntry[2] if len(asEntry) > 2 and asEntry[2] else 'misc/other'; + sDescription = asEntry[3] if len(asEntry) > 3 and asEntry[3] else ''; + sAltName = asEntry[4] if len(asEntry) > 4 and asEntry[4] else None; + assert len(asEntry) <= 5 and sGstFile and sHstFile; + if not os.path.isabs(sHstFile): + sHstFile = os.path.join(self.sScratchPath, sHstFile); + + reporter.log2('Downloading file "%s" to "%s" ...' % (sGstFile, sHstFile,)); + + try: os.unlink(sHstFile); ## @todo txsDownloadFile doesn't truncate the output file. + except: pass; + + fRc = self.txsDownloadFile(oSession, oTxsSession, sGstFile, sHstFile, 30 * 1000, fIgnoreErrors); + if fRc: + if fAddToLog: + reporter.addLogFile(sHstFile, sKind, sDescription, sAltName); + else: + if fIgnoreErrors is not True: + return reporter.error('error downloading file "%s" to "%s"' % (sGstFile, sHstFile)); + reporter.log('warning: file "%s" was not downloaded, ignoring.' % (sGstFile,)); + return True; + + def txsDownloadString(self, oSession, oTxsSession, sRemoteFile, sEncoding = 'utf-8', fIgnoreEncodingErrors = True, + cMsTimeout = 30000, fIgnoreErrors = False): + return self.txsDoTask(oSession, oTxsSession, oTxsSession.asyncDownloadString, + (sRemoteFile, sEncoding, fIgnoreEncodingErrors, self.adjustTimeoutMs(cMsTimeout), fIgnoreErrors)); + + def txsPackFile(self, oSession, oTxsSession, sRemoteFile, sRemoteSource, cMsTimeout = 30000, fIgnoreErrors = False): + return self.txsDoTask(oSession, oTxsSession, oTxsSession.asyncPackFile, \ + (sRemoteFile, sRemoteSource, self.adjustTimeoutMs(cMsTimeout), fIgnoreErrors)); + + def txsUnpackFile(self, oSession, oTxsSession, sRemoteFile, sRemoteDir, cMsTimeout = 30000, fIgnoreErrors = False): + return self.txsDoTask(oSession, oTxsSession, oTxsSession.asyncUnpackFile, \ + (sRemoteFile, sRemoteDir, self.adjustTimeoutMs(cMsTimeout), fIgnoreErrors)); + + def txsExpandString(self, oSession, oTxsSession, sString, cMsTimeout = 30000, fIgnoreErrors = False): + return self.txsDoTask(oSession, oTxsSession, oTxsSession.asyncExpandString, \ + (sString, self.adjustTimeoutMs(cMsTimeout), fIgnoreErrors)); + + # pylint: enable=missing-docstring + + def txsCdWait(self, + oSession, # type: vboxwrappers.SessionWrapper + oTxsSession, # type: txsclient.Session + cMsTimeout = 30000, # type: int + sFile = None # type: String + ): # -> bool + """ + Mostly an internal helper for txsRebootAndReconnectViaTcp and + startVmAndConnectToTxsViaTcp that waits for the CDROM drive to become + ready. It does this by polling for a file it knows to exist on the CD. + + Returns True on success. + + Returns False on failure, logged. + """ + + if sFile is None: + sFile = 'valkit.txt'; + + reporter.log('txsCdWait: Waiting for file "%s" to become available ...' % (sFile,)); + + fRemoveVm = self.addTask(oSession); + fRemoveTxs = self.addTask(oTxsSession); + cMsTimeout = self.adjustTimeoutMs(cMsTimeout); + msStart = base.timestampMilli(); + cMsTimeout2 = cMsTimeout; + fRc = oTxsSession.asyncIsFile('${CDROM}/%s' % (sFile,), cMsTimeout2); + if fRc is True: + while True: + # wait for it to complete. + oTask = self.waitForTasks(cMsTimeout2 + 1); + if oTask is not oTxsSession: + oTxsSession.cancelTask(); + if oTask is None: + reporter.errorTimeout('txsCdWait: The task timed out (after %s ms).' + % (base.timestampMilli() - msStart,)); + elif oTask is oSession: + reporter.error('txsCdWait: The VM terminated unexpectedly'); + else: + reporter.error('txsCdWait: An unknown task %s was returned' % (oTask,)); + fRc = False; + break; + if oTxsSession.isSuccess(): + break; + + # Check for timeout. + cMsElapsed = base.timestampMilli() - msStart; + if cMsElapsed >= cMsTimeout: + reporter.error('txsCdWait: timed out'); + fRc = False; + break; + # delay. + self.sleep(1); + + # resubmit the task. + cMsTimeout2 = msStart + cMsTimeout - base.timestampMilli(); + cMsTimeout2 = max(cMsTimeout2, 500); + fRc = oTxsSession.asyncIsFile('${CDROM}/%s' % (sFile,), cMsTimeout2); + if fRc is not True: + reporter.error('txsCdWait: asyncIsFile failed'); + break; + else: + reporter.error('txsCdWait: asyncIsFile failed'); + + if not fRc: + # Do some diagnosis to find out why this failed. + ## @todo Identify guest OS type and only run one of the following commands. + fIsNotWindows = True; + reporter.log('txsCdWait: Listing root contents of ${CDROM}:'); + if fIsNotWindows: + reporter.log('txsCdWait: Tiggering udevadm ...'); + oTxsSession.syncExec("/sbin/udevadm", ("/sbin/udevadm", "trigger", "--verbose"), fIgnoreErrors = True); + time.sleep(15); + oTxsSession.syncExec("/bin/ls", ("/bin/ls", "-al", "${CDROM}"), fIgnoreErrors = True); + reporter.log('txsCdWait: Listing media directory:'); + oTxsSession.syncExec('/bin/ls', ('/bin/ls', '-l', '-a', '-R', '/media'), fIgnoreErrors = True); + reporter.log('txsCdWait: Listing mount points / drives:'); + oTxsSession.syncExec('/bin/mount', ('/bin/mount',), fIgnoreErrors = True); + oTxsSession.syncExec('/bin/cat', ('/bin/cat', '/etc/fstab'), fIgnoreErrors = True); + oTxsSession.syncExec('/bin/dmesg', ('/bin/dmesg',), fIgnoreErrors = True); + oTxsSession.syncExec('/usr/bin/lshw', ('/usr/bin/lshw', '-c', 'disk'), fIgnoreErrors = True); + oTxsSession.syncExec('/bin/journalctl', + ('/bin/journalctl', '-x', '-b'), fIgnoreErrors = True); + oTxsSession.syncExec('/bin/journalctl', + ('/bin/journalctl', '-x', '-b', '/usr/lib/udisks2/udisksd'), fIgnoreErrors = True); + oTxsSession.syncExec('/usr/bin/udisksctl', + ('/usr/bin/udisksctl', 'info', '-b', '/dev/sr0'), fIgnoreErrors = True); + oTxsSession.syncExec('/bin/systemctl', + ('/bin/systemctl', 'status', 'udisks2'), fIgnoreErrors = True); + oTxsSession.syncExec('/bin/ps', + ('/bin/ps', '-a', '-u', '-x'), fIgnoreErrors = True); + reporter.log('txsCdWait: Mounting manually ...'); + for _ in range(3): + oTxsSession.syncExec('/bin/mount', ('/bin/mount', '/dev/sr0', '${CDROM}'), fIgnoreErrors = True); + time.sleep(5); + reporter.log('txsCdWait: Re-Listing media directory:'); + oTxsSession.syncExec('/bin/ls', ('/bin/ls', '-l', '-a', '-R', '/media'), fIgnoreErrors = True); + else: + # ASSUMES that we always install Windows on drive C right now. + sWinDir = "C:\\Windows\\System32\\"; + # Should work since WinXP Pro. + oTxsSession.syncExec(sWinDir + "wbem\\WMIC.exe", + ("WMIC.exe", "logicaldisk", "get", + "deviceid, volumename, description"), + fIgnoreErrors = True); + oTxsSession.syncExec(sWinDir + " cmd.exe", + ('cmd.exe', '/C', 'dir', '${CDROM}'), + fIgnoreErrors = True); + + if fRemoveTxs: + self.removeTask(oTxsSession); + if fRemoveVm: + self.removeTask(oSession); + return fRc; + + def txsDoConnectViaTcp(self, oSession, cMsTimeout, fNatForwardingForTxs = False): + """ + Mostly an internal worker for connecting to TXS via TCP used by the + *ViaTcp methods. + + Returns a tuplet with True/False and TxsSession/None depending on the + result. Errors are logged. + """ + + reporter.log2('txsDoConnectViaTcp: oSession=%s, cMsTimeout=%s, fNatForwardingForTxs=%s' + % (oSession, cMsTimeout, fNatForwardingForTxs)); + + cMsTimeout = self.adjustTimeoutMs(cMsTimeout); + oTxsConnect = oSession.txsConnectViaTcp(cMsTimeout, fNatForwardingForTxs = fNatForwardingForTxs); + if oTxsConnect is not None: + self.addTask(oTxsConnect); + fRemoveVm = self.addTask(oSession); + oTask = self.waitForTasks(cMsTimeout + 1); + reporter.log2('txsDoConnectViaTcp: waitForTasks returned %s' % (oTask,)); + self.removeTask(oTxsConnect); + if oTask is oTxsConnect: + oTxsSession = oTxsConnect.getResult(); + if oTxsSession is not None: + reporter.log('txsDoConnectViaTcp: Connected to TXS on %s.' % (oTxsSession.oTransport.sHostname,)); + return (True, oTxsSession); + + reporter.error('txsDoConnectViaTcp: failed to connect to TXS.'); + else: + oTxsConnect.cancelTask(); + if oTask is None: + reporter.errorTimeout('txsDoConnectViaTcp: connect stage 1 timed out'); + elif oTask is oSession: + oSession.reportPrematureTermination('txsDoConnectViaTcp: '); + else: + reporter.error('txsDoConnectViaTcp: unknown/wrong task %s' % (oTask,)); + if fRemoveVm: + self.removeTask(oSession); + else: + reporter.error('txsDoConnectViaTcp: txsConnectViaTcp failed'); + return (False, None); + + def startVmAndConnectToTxsViaTcp(self, sVmName, fCdWait = False, cMsTimeout = 15*60000, \ + cMsCdWait = 30000, sFileCdWait = None, \ + fNatForwardingForTxs = False): + """ + Starts the specified VM and tries to connect to its TXS via TCP. + The VM will be powered off if TXS doesn't respond before the specified + time has elapsed. + + Returns a the VM and TXS sessions (a two tuple) on success. The VM + session is in the task list, the TXS session is not. + Returns (None, None) on failure, fully logged. + """ + + # Zap the guest IP to make sure we're not getting a stale entry + # (unless we're restoring the VM of course). + oTestVM = self.oTestVmSet.findTestVmByName(sVmName) if self.oTestVmSet is not None else None; + if oTestVM is None \ + or oTestVM.fSnapshotRestoreCurrent is False: + try: + oSession1 = self.openSession(self.getVmByName(sVmName)); + oSession1.delGuestPropertyValue('/VirtualBox/GuestInfo/Net/0/V4/IP'); + oSession1.saveSettings(True); + del oSession1; + except: + reporter.logXcpt(); + + # Start the VM. + reporter.log('startVmAndConnectToTxsViaTcp: Starting(/preparing) "%s" (timeout %s s)...' % (sVmName, cMsTimeout / 1000)); + reporter.flushall(); + oSession = self.startVmByName(sVmName); + if oSession is not None: + # Connect to TXS. + reporter.log2('startVmAndConnectToTxsViaTcp: Started(/prepared) "%s", connecting to TXS ...' % (sVmName,)); + (fRc, oTxsSession) = self.txsDoConnectViaTcp(oSession, cMsTimeout, fNatForwardingForTxs); + if fRc is True: + if fCdWait: + # Wait for CD? + reporter.log2('startVmAndConnectToTxsViaTcp: Waiting for file "%s" to become available ...' % (sFileCdWait,)); + fRc = self.txsCdWait(oSession, oTxsSession, cMsCdWait, sFileCdWait); + if fRc is not True: + reporter.error('startVmAndConnectToTxsViaTcp: txsCdWait failed'); + + sVer = self.txsVer(oSession, oTxsSession, cMsTimeout, fIgnoreErrors = True); + if sVer is not False: + reporter.log('startVmAndConnectToTxsViaTcp: TestExecService version %s' % (sVer,)); + else: + reporter.log('startVmAndConnectToTxsViaTcp: Unable to retrieve TestExecService version'); + + if fRc is True: + # Success! + return (oSession, oTxsSession); + else: + reporter.error('startVmAndConnectToTxsViaTcp: txsDoConnectViaTcp failed'); + # If something went wrong while waiting for TXS to be started - take VM screenshot before terminate it + self.terminateVmBySession(oSession); + return (None, None); + + def txsRebootAndReconnectViaTcp(self, oSession, oTxsSession, fCdWait = False, cMsTimeout = 15*60000, \ + cMsCdWait = 30000, sFileCdWait = None, fNatForwardingForTxs = False): + """ + Executes the TXS reboot command + + Returns A tuple of True and the new TXS session on success. + + Returns A tuple of False and either the old TXS session or None on failure. + """ + reporter.log2('txsRebootAndReconnect: cMsTimeout=%u' % (cMsTimeout,)); + + # + # This stuff is a bit complicated because of rebooting being kind of + # disruptive to the TXS and such... The protocol is that TXS will: + # - ACK the reboot command. + # - Shutdown the transport layer, implicitly disconnecting us. + # - Execute the reboot operation. + # - On failure, it will be re-init the transport layer and be + # available pretty much immediately. UUID unchanged. + # - On success, it will be respawed after the reboot (hopefully), + # with a different UUID. + # + fRc = False; + iStart = base.timestampMilli(); + + # Get UUID. + cMsTimeout2 = min(60000, cMsTimeout); + sUuidBefore = self.txsUuid(oSession, oTxsSession, self.adjustTimeoutMs(cMsTimeout2, 60000)); + if sUuidBefore is not False: + # Reboot. + cMsElapsed = base.timestampMilli() - iStart; + cMsTimeout2 = cMsTimeout - cMsElapsed; + fRc = self.txsDoTask(oSession, oTxsSession, oTxsSession.asyncReboot, + (self.adjustTimeoutMs(cMsTimeout2, 60000), False)); + if fRc is True: + # Reconnect. + if fNatForwardingForTxs is True: + self.sleep(22); # NAT fudge - Two fixes are wanted: 1. TXS connect retries. 2. Main API reboot/reset hint. + cMsElapsed = base.timestampMilli() - iStart; + (fRc, oTxsSession) = self.txsDoConnectViaTcp(oSession, cMsTimeout - cMsElapsed, fNatForwardingForTxs); + if fRc is True: + # Check the UUID. + cMsElapsed = base.timestampMilli() - iStart; + cMsTimeout2 = min(60000, cMsTimeout - cMsElapsed); + sUuidAfter = self.txsDoTask(oSession, oTxsSession, oTxsSession.asyncUuid, + (self.adjustTimeoutMs(cMsTimeout2, 60000), False)); + if sUuidBefore is not False: + if sUuidAfter != sUuidBefore: + reporter.log('The guest rebooted (UUID %s -> %s)' % (sUuidBefore, sUuidAfter)) + + # Do CD wait if specified. + if fCdWait: + fRc = self.txsCdWait(oSession, oTxsSession, cMsCdWait, sFileCdWait); + if fRc is not True: + reporter.error('txsRebootAndReconnectViaTcp: txsCdWait failed'); + + sVer = self.txsVer(oSession, oTxsSession, cMsTimeout, fIgnoreErrors = True); + if sVer is not False: + reporter.log('txsRebootAndReconnectViaTcp: TestExecService version %s' % (sVer,)); + else: + reporter.log('txsRebootAndReconnectViaTcp: Unable to retrieve TestExecService version'); + else: + reporter.error('txsRebootAndReconnectViaTcp: failed to get UUID (after)'); + else: + reporter.error('txsRebootAndReconnectViaTcp: did not reboot (UUID %s)' % (sUuidBefore,)); + else: + reporter.error('txsRebootAndReconnectViaTcp: txsDoConnectViaTcp failed'); + else: + reporter.error('txsRebootAndReconnectViaTcp: reboot failed'); + else: + reporter.error('txsRebootAndReconnectViaTcp: failed to get UUID (before)'); + return (fRc, oTxsSession); + + # pylint: disable=too-many-locals,too-many-arguments + + def txsRunTest(self, oTxsSession, sTestName, cMsTimeout, sExecName, asArgs = (), asAddEnv = (), sAsUser = "", + fCheckSessionStatus = False): + """ + Executes the specified test task, waiting till it completes or times out. + + The VM session (if any) must be in the task list. + + Returns True if we executed the task and nothing abnormal happend. + Query the process status from the TXS session. + + Returns False if some unexpected task was signalled or we failed to + submit the job. + + If fCheckSessionStatus is set to True, the overall session status will be + taken into account and logged as an error on failure. + """ + reporter.testStart(sTestName); + reporter.log2('txsRunTest: cMsTimeout=%u sExecName=%s asArgs=%s' % (cMsTimeout, sExecName, asArgs)); + + # Submit the job. + fRc = False; + if oTxsSession.asyncExec(sExecName, asArgs, asAddEnv, sAsUser, cMsTimeout = self.adjustTimeoutMs(cMsTimeout)): + self.addTask(oTxsSession); + + # Wait for the job to complete. + while True: + oTask = self.waitForTasks(cMsTimeout + 1); + if oTask is None: + if fCheckSessionStatus: + reporter.error('txsRunTest: waitForTasks for test "%s" timed out' % (sTestName,)); + else: + reporter.log('txsRunTest: waitForTasks for test "%s" timed out' % (sTestName,)); + break; + if oTask is oTxsSession: + if fCheckSessionStatus \ + and not oTxsSession.isSuccess(): + reporter.error('txsRunTest: Test "%s" failed' % (sTestName,)); + else: + fRc = True; + reporter.log('txsRunTest: isSuccess=%s getResult=%s' \ + % (oTxsSession.isSuccess(), oTxsSession.getResult())); + break; + if not self.handleTask(oTask, 'txsRunTest'): + break; + + self.removeTask(oTxsSession); + if not oTxsSession.pollTask(): + oTxsSession.cancelTask(); + else: + reporter.error('txsRunTest: asyncExec failed'); + + reporter.testDone(); + return fRc; + + def txsRunTestRedirectStd(self, oTxsSession, sTestName, cMsTimeout, sExecName, asArgs = (), asAddEnv = (), sAsUser = "", + oStdIn = '/dev/null', oStdOut = '/dev/null', oStdErr = '/dev/null', oTestPipe = '/dev/null'): + """ + Executes the specified test task, waiting till it completes or times out, + redirecting stdin, stdout and stderr to the given objects. + + The VM session (if any) must be in the task list. + + Returns True if we executed the task and nothing abnormal happend. + Query the process status from the TXS session. + + Returns False if some unexpected task was signalled or we failed to + submit the job. + """ + reporter.testStart(sTestName); + reporter.log2('txsRunTestRedirectStd: cMsTimeout=%u sExecName=%s asArgs=%s' % (cMsTimeout, sExecName, asArgs)); + + # Submit the job. + fRc = False; + if oTxsSession.asyncExecEx(sExecName, asArgs, asAddEnv, oStdIn, oStdOut, oStdErr, + oTestPipe, sAsUser, cMsTimeout = self.adjustTimeoutMs(cMsTimeout)): + self.addTask(oTxsSession); + + # Wait for the job to complete. + while True: + oTask = self.waitForTasks(cMsTimeout + 1); + if oTask is None: + reporter.log('txsRunTestRedirectStd: waitForTasks timed out'); + break; + if oTask is oTxsSession: + fRc = True; + reporter.log('txsRunTestRedirectStd: isSuccess=%s getResult=%s' + % (oTxsSession.isSuccess(), oTxsSession.getResult())); + break; + if not self.handleTask(oTask, 'txsRunTestRedirectStd'): + break; + + self.removeTask(oTxsSession); + if not oTxsSession.pollTask(): + oTxsSession.cancelTask(); + else: + reporter.error('txsRunTestRedirectStd: asyncExec failed'); + + reporter.testDone(); + return fRc; + + def txsRunTest2(self, oTxsSession1, oTxsSession2, sTestName, cMsTimeout, + sExecName1, asArgs1, + sExecName2, asArgs2, + asAddEnv1 = (), sAsUser1 = '', fWithTestPipe1 = True, + asAddEnv2 = (), sAsUser2 = '', fWithTestPipe2 = True): + """ + Executes the specified test tasks, waiting till they complete or + times out. The 1st task is started after the 2nd one. + + The VM session (if any) must be in the task list. + + Returns True if we executed the task and nothing abnormal happend. + Query the process status from the TXS sessions. + + Returns False if some unexpected task was signalled or we failed to + submit the job. + """ + reporter.testStart(sTestName); + + # Submit the jobs. + fRc = False; + if oTxsSession1.asyncExec(sExecName1, asArgs1, asAddEnv1, sAsUser1, fWithTestPipe1, '1-', + self.adjustTimeoutMs(cMsTimeout)): + self.addTask(oTxsSession1); + + self.sleep(2); # fudge! grr + + if oTxsSession2.asyncExec(sExecName2, asArgs2, asAddEnv2, sAsUser2, fWithTestPipe2, '2-', + self.adjustTimeoutMs(cMsTimeout)): + self.addTask(oTxsSession2); + + # Wait for the jobs to complete. + cPendingJobs = 2; + while True: + oTask = self.waitForTasks(cMsTimeout + 1); + if oTask is None: + reporter.log('txsRunTest2: waitForTasks timed out'); + break; + + if oTask is oTxsSession1 or oTask is oTxsSession2: + if oTask is oTxsSession1: iTask = 1; + else: iTask = 2; + reporter.log('txsRunTest2: #%u - isSuccess=%s getResult=%s' \ + % (iTask, oTask.isSuccess(), oTask.getResult())); + self.removeTask(oTask); + cPendingJobs -= 1; + if cPendingJobs <= 0: + fRc = True; + break; + + elif not self.handleTask(oTask, 'txsRunTest'): + break; + + self.removeTask(oTxsSession2); + if not oTxsSession2.pollTask(): + oTxsSession2.cancelTask(); + else: + reporter.error('txsRunTest2: asyncExec #2 failed'); + + self.removeTask(oTxsSession1); + if not oTxsSession1.pollTask(): + oTxsSession1.cancelTask(); + else: + reporter.error('txsRunTest2: asyncExec #1 failed'); + + reporter.testDone(); + return fRc; + + # pylint: enable=too-many-locals,too-many-arguments + + + # + # Working with test results via serial port. + # + + class TxsMonitorComFile(base.TdTaskBase): + """ + Class that monitors a COM output file. + """ + + def __init__(self, sComRawFile, asStopWords = None): + base.TdTaskBase.__init__(self, utils.getCallerName()); + self.sComRawFile = sComRawFile; + self.oStopRegExp = re.compile('\\b(' + '|'.join(asStopWords if asStopWords else ('PASSED', 'FAILED',)) + ')\\b'); + self.sResult = None; ##< The result. + self.cchDisplayed = 0; ##< Offset into the file string of what we've already fed to the logger. + + def toString(self): + return '<%s sComRawFile=%s oStopRegExp=%s sResult=%s cchDisplayed=%s>' \ + % (base.TdTaskBase.toString(self), self.sComRawFile, self.oStopRegExp, self.sResult, self.cchDisplayed,); + + def pollTask(self, fLocked = False): + """ + Overrides TdTaskBase.pollTask() for the purpose of polling the file. + """ + if not fLocked: + self.lockTask(); + + sFile = utils.noxcptReadFile(self.sComRawFile, '', 'rU'); + if len(sFile) > self.cchDisplayed: + sNew = sFile[self.cchDisplayed:]; + oMatch = self.oStopRegExp.search(sNew); + if oMatch: + # Done! Get result, flush all the output and signal the task. + self.sResult = oMatch.group(1); + for sLine in sNew.split('\n'): + reporter.log('COM OUTPUT: %s' % (sLine,)); + self.cchDisplayed = len(sFile); + self.signalTaskLocked(); + else: + # Output whole lines only. + offNewline = sFile.find('\n', self.cchDisplayed); + while offNewline >= 0: + reporter.log('COM OUTPUT: %s' % (sFile[self.cchDisplayed:offNewline])) + self.cchDisplayed = offNewline + 1; + offNewline = sFile.find('\n', self.cchDisplayed); + + fRet = self.fSignalled; + if not fLocked: + self.unlockTask(); + return fRet; + + # Our stuff. + def getResult(self): + """ + Returns the connected TXS session object on success. + Returns None on failure or if the task has not yet completed. + """ + self.oCv.acquire(); + sResult = self.sResult; + self.oCv.release(); + return sResult; + + def cancelTask(self): + """ Cancels the task. """ + self.signalTask(); + return True; + + + def monitorComRawFile(self, oSession, sComRawFile, cMsTimeout = 15*60000, asStopWords = None): + """ + Monitors the COM output file for stop words (PASSED and FAILED by default). + + Returns the stop word. + Returns None on VM error and timeout. + """ + + reporter.log2('monitorComRawFile: oSession=%s, cMsTimeout=%s, sComRawFile=%s' % (oSession, cMsTimeout, sComRawFile)); + + oMonitorTask = self.TxsMonitorComFile(sComRawFile, asStopWords); + self.addTask(oMonitorTask); + + cMsTimeout = self.adjustTimeoutMs(cMsTimeout); + oTask = self.waitForTasks(cMsTimeout + 1); + reporter.log2('monitorComRawFile: waitForTasks returned %s' % (oTask,)); + + if oTask is not oMonitorTask: + oMonitorTask.cancelTask(); + self.removeTask(oMonitorTask); + + oMonitorTask.pollTask(); + return oMonitorTask.getResult(); + + + def runVmAndMonitorComRawFile(self, sVmName, sComRawFile, cMsTimeout = 15*60000, asStopWords = None): + """ + Runs the specified VM and monitors the given COM output file for stop + words (PASSED and FAILED by default). + + The caller is assumed to have configured the VM to use the given + file. The method will take no action to verify this. + + Returns the stop word. + Returns None on VM error and timeout. + """ + + # Start the VM. + reporter.log('runVmAndMonitorComRawFile: Starting(/preparing) "%s" (timeout %s s)...' % (sVmName, cMsTimeout / 1000)); + reporter.flushall(); + oSession = self.startVmByName(sVmName); + if oSession is not None: + # Let it run and then terminate it. + sRet = self.monitorComRawFile(oSession, sComRawFile, cMsTimeout, asStopWords); + self.terminateVmBySession(oSession); + else: + sRet = None; + return sRet; + + # + # Other stuff + # + + def waitForGAs(self, + oSession, # type: vboxwrappers.SessionWrapper + cMsTimeout = 120000, aenmWaitForRunLevels = None, aenmWaitForActive = None, aenmWaitForInactive = None): + """ + Waits for the guest additions to enter a certain state. + + aenmWaitForRunLevels - List of run level values to wait for (success if one matches). + aenmWaitForActive - List facilities (type values) that must be active. + aenmWaitForInactive - List facilities (type values) that must be inactive. + + Defaults to wait for AdditionsRunLevelType_Userland if nothing else is given. + + Returns True on success, False w/ error logging on timeout or failure. + """ + reporter.log2('waitForGAs: oSession=%s, cMsTimeout=%s' % (oSession, cMsTimeout,)); + + # + # Get IGuest: + # + try: + oIGuest = oSession.o.console.guest; + except: + return reporter.errorXcpt(); + + # + # Create a wait task: + # + from testdriver.vboxwrappers import AdditionsStatusTask; + try: + oGaStatusTask = AdditionsStatusTask(oSession = oSession, + oIGuest = oIGuest, + cMsTimeout = cMsTimeout, + aenmWaitForRunLevels = aenmWaitForRunLevels, + aenmWaitForActive = aenmWaitForActive, + aenmWaitForInactive = aenmWaitForInactive); + except: + return reporter.errorXcpt(); + + # + # Add the task and make sure the VM session is also present. + # + self.addTask(oGaStatusTask); + fRemoveSession = self.addTask(oSession); + oTask = self.waitForTasks(cMsTimeout + 1); + reporter.log2('waitForGAs: returned %s (oGaStatusTask=%s, oSession=%s)' % (oTask, oGaStatusTask, oSession,)); + self.removeTask(oGaStatusTask); + if fRemoveSession: + self.removeTask(oSession); + + # + # Digest the result. + # + if oTask is oGaStatusTask: + fSucceeded = oGaStatusTask.getResult(); + if fSucceeded is True: + reporter.log('waitForGAs: Succeeded.'); + else: + reporter.error('waitForGAs: Failed.'); + else: + oGaStatusTask.cancelTask(); + if oTask is None: + reporter.error('waitForGAs: Timed out.'); + elif oTask is oSession: + oSession.reportPrematureTermination('waitForGAs: '); + else: + reporter.error('waitForGAs: unknown/wrong task %s' % (oTask,)); + fSucceeded = False; + return fSucceeded; + + @staticmethod + def controllerTypeToName(eControllerType): + """ + Translate a controller type to a standard controller name. + """ + if eControllerType in (vboxcon.StorageControllerType_PIIX3, vboxcon.StorageControllerType_PIIX4,): + sName = "IDE Controller"; + elif eControllerType == vboxcon.StorageControllerType_IntelAhci: + sName = "SATA Controller"; + elif eControllerType == vboxcon.StorageControllerType_LsiLogicSas: + sName = "SAS Controller"; + elif eControllerType in (vboxcon.StorageControllerType_LsiLogic, vboxcon.StorageControllerType_BusLogic,): + sName = "SCSI Controller"; + elif eControllerType == vboxcon.StorageControllerType_NVMe: + sName = "NVMe Controller"; + elif eControllerType == vboxcon.StorageControllerType_VirtioSCSI: + sName = "VirtIO SCSI Controller"; + else: + sName = "Storage Controller"; + return sName; diff --git a/src/VBox/ValidationKit/testdriver/vboxcon.py b/src/VBox/ValidationKit/testdriver/vboxcon.py new file mode 100755 index 00000000..30b83dd2 --- /dev/null +++ b/src/VBox/ValidationKit/testdriver/vboxcon.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- +# $Id: vboxcon.py $ + +""" +VirtualBox Constants. + +See VBoxConstantWrappingHack for details. +""" + +__copyright__ = \ +""" +Copyright (C) 2010-2022 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 153224 $" + + +# Standard Python imports. +import sys + + +class VBoxConstantWrappingHack(object): # pylint: disable=too-few-public-methods + """ + This is a hack to avoid the self.oVBoxMgr.constants.MachineState_Running + ugliness that forces one into the right margin... Anyone using this module + can get to the constants easily by: + + from testdriver import vboxcon + if self.o.machine.state == vboxcon.MachineState_Running: + do stuff; + + For our own convenience there's a vboxcon attribute set up in vbox.py, + class TestDriver which is the basis for the VirtualBox testcases. It takes + care of setting things up properly through the global variable + 'goHackModuleClass' that refers to the instance of this class(if we didn't + we'd have to use testdriver.vboxcon.MachineState_Running). + """ + def __init__(self, oWrapped): + self.oWrapped = oWrapped; + self.oVBoxMgr = None; + self.fpApiVer = 99.0; + + def __getattr__(self, sName): + # Our self. + try: + return getattr(self.oWrapped, sName) + except AttributeError: + # The VBox constants. + if self.oVBoxMgr is None: + raise; + try: + return getattr(self.oVBoxMgr.constants, sName); + except AttributeError: + # Do some compatability mappings to keep it working with + # older versions. + if self.fpApiVer < 3.3: + if sName == 'SessionState_Locked': + return getattr(self.oVBoxMgr.constants, 'SessionState_Open'); + if sName == 'SessionState_Unlocked': + return getattr(self.oVBoxMgr.constants, 'SessionState_Closed'); + if sName == 'SessionState_Unlocking': + return getattr(self.oVBoxMgr.constants, 'SessionState_Closing'); + raise; + + +goHackModuleClass = VBoxConstantWrappingHack(sys.modules[__name__]); # pylint: disable=invalid-name +sys.modules[__name__] = goHackModuleClass; + diff --git a/src/VBox/ValidationKit/testdriver/vboxinstaller.py b/src/VBox/ValidationKit/testdriver/vboxinstaller.py new file mode 100755 index 00000000..8d973bf1 --- /dev/null +++ b/src/VBox/ValidationKit/testdriver/vboxinstaller.py @@ -0,0 +1,1251 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +VirtualBox Installer Wrapper Driver. + +This installs VirtualBox, starts a sub driver which does the real testing, +and then uninstall VirtualBox afterwards. This reduces the complexity of the +other VBox test drivers. +""" + +__copyright__ = \ +""" +Copyright (C) 2010-2022 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155126 $" + + +# Standard Python imports. +import os +import sys +import re +import socket +import tempfile +import time + +# Only the main script needs to modify the path. +try: __file__ +except: __file__ = sys.argv[0]; +g_ksValidationKitDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))); +sys.path.append(g_ksValidationKitDir); + +# Validation Kit imports. +from common import utils, webutils; +from common.constants import rtexitcode; +from testdriver import reporter; +from testdriver.base import TestDriverBase; + + + +class VBoxInstallerTestDriver(TestDriverBase): + """ + Implementation of a top level test driver. + """ + + + ## State file indicating that we've skipped installation. + ksVar_Skipped = 'vboxinstaller-skipped'; + + + def __init__(self): + TestDriverBase.__init__(self); + self._asSubDriver = []; # The sub driver and it's arguments. + self._asBuildUrls = []; # The URLs passed us on the command line. + self._asBuildFiles = []; # The downloaded file names. + self._fUnpackedBuildFiles = False; + self._fAutoInstallPuelExtPack = True; + self._fKernelDrivers = True; + self._fWinForcedInstallTimestampCA = True; + self._fInstallMsCrt = False; # By default we don't install the Microsoft CRT (only needed once). + + # + # Base method we override + # + + def showUsage(self): + rc = TestDriverBase.showUsage(self); + # 0 1 2 3 4 5 6 7 8 + # 012345678901234567890123456789012345678901234567890123456789012345678901234567890 + reporter.log(''); + reporter.log('vboxinstaller Options:'); + reporter.log(' --vbox-build <url[,url2[,...]]>'); + reporter.log(' Comma separated list of URL to file to download and install or/and'); + reporter.log(' unpack. URLs without a schema are assumed to be files on the'); + reporter.log(' build share and will be copied off it.'); + reporter.log(' --no-puel-extpack'); + reporter.log(' Indicates that the PUEL extension pack should not be installed if found.'); + reporter.log(' The default is to install it when found in the vbox-build.'); + reporter.log(' --no-kernel-drivers'); + reporter.log(' Indicates that the kernel drivers should not be installed on platforms'); + reporter.log(' where this is optional. The default is to install them.'); + reporter.log(' --forced-win-install-timestamp-ca, --no-forced-win-install-timestamp-ca'); + reporter.log(' Whether to force installation of the legacy Windows timestamp CA.'); + reporter.log(' If not forced, it will only installed on the hosts that needs it.'); + reporter.log(' Default: --no-forced-win-install-timestamp-ca'); + reporter.log(' --win-install-mscrt, --no-win-install-mscrt'); + reporter.log(' Whether to install the MS Visual Studio Redistributable.'); + reporter.log(' Default: --no-win-install-mscrt'); + reporter.log(' --'); + reporter.log(' Indicates the end of our parameters and the start of the sub'); + reporter.log(' testdriver and its arguments.'); + return rc; + + def parseOption(self, asArgs, iArg): + """ + Parse our arguments. + """ + if asArgs[iArg] == '--': + # End of our parameters and start of the sub driver invocation. + iArg = self.requireMoreArgs(1, asArgs, iArg); + assert not self._asSubDriver; + self._asSubDriver = asArgs[iArg:]; + self._asSubDriver[0] = self._asSubDriver[0].replace('/', os.path.sep); + iArg = len(asArgs) - 1; + elif asArgs[iArg] == '--vbox-build': + # List of files to copy/download and install. + iArg = self.requireMoreArgs(1, asArgs, iArg); + self._asBuildUrls = asArgs[iArg].split(','); + elif asArgs[iArg] == '--no-puel-extpack': + self._fAutoInstallPuelExtPack = False; + elif asArgs[iArg] == '--puel-extpack': + self._fAutoInstallPuelExtPack = True; + elif asArgs[iArg] == '--no-kernel-drivers': + self._fKernelDrivers = False; + elif asArgs[iArg] == '--kernel-drivers': + self._fKernelDrivers = True; + elif asArgs[iArg] == '--no-forced-win-install-timestamp-ca': + self._fWinForcedInstallTimestampCA = False; + elif asArgs[iArg] == '--forced-win-install-timestamp-ca': + self._fWinForcedInstallTimestampCA = True; + elif asArgs[iArg] == '--no-win-install-mscrt': + self._fInstallMsCrt = False; + elif asArgs[iArg] == '--win-install-mscrt': + self._fInstallMsCrt = True; + else: + return TestDriverBase.parseOption(self, asArgs, iArg); + return iArg + 1; + + def completeOptions(self): + # + # Check that we've got what we need. + # + if not self._asBuildUrls: + reporter.error('No build files specified ("--vbox-build file1[,file2[...]]")'); + return False; + if not self._asSubDriver: + reporter.error('No sub testdriver specified. (" -- test/stuff/tdStuff1.py args")'); + return False; + + # + # Construct _asBuildFiles as an array parallel to _asBuildUrls. + # + for sUrl in self._asBuildUrls: + sDstFile = os.path.join(self.sScratchPath, webutils.getFilename(sUrl)); + self._asBuildFiles.append(sDstFile); + + return TestDriverBase.completeOptions(self); + + def actionExtract(self): + reporter.error('vboxinstall does not support extracting resources, you have to do that using the sub testdriver.'); + return False; + + def actionCleanupBefore(self): + """ + Kills all VBox process we see. + + This is only supposed to execute on a testbox so we don't need to go + all complicated wrt other users. + """ + return self._killAllVBoxProcesses(); + + def actionConfig(self): + """ + Install VBox and pass on the configure request to the sub testdriver. + """ + fRc = self._installVBox(); + if fRc is None: + self._persistentVarSet(self.ksVar_Skipped, 'true'); + self.fBadTestbox = True; + else: + self._persistentVarUnset(self.ksVar_Skipped); + + ## @todo vbox.py still has bugs preventing us from invoking it seperately with each action. + if fRc is True and 'execute' not in self.asActions and 'all' not in self.asActions: + fRc = self._executeSubDriver([ 'verify', ]); + if fRc is True and 'execute' not in self.asActions and 'all' not in self.asActions: + fRc = self._executeSubDriver([ 'config', ], fPreloadASan = True); + return fRc; + + def actionExecute(self): + """ + Execute the sub testdriver. + """ + return self._executeSubDriver(self.asActions, fPreloadASan = True); + + def actionCleanupAfter(self): + """ + Forward this to the sub testdriver, then uninstall VBox. + """ + fRc = True; + if 'execute' not in self.asActions and 'all' not in self.asActions: + fRc = self._executeSubDriver([ 'cleanup-after', ], fMaySkip = False); + + if not self._killAllVBoxProcesses(): + fRc = False; + + if not self._uninstallVBox(self._persistentVarExists(self.ksVar_Skipped)): + fRc = False; + + if utils.getHostOs() == 'darwin': + self._darwinUnmountDmg(fIgnoreError = True); # paranoia + + if not TestDriverBase.actionCleanupAfter(self): + fRc = False; + + return fRc; + + + def actionAbort(self): + """ + Forward this to the sub testdriver first, then wipe all VBox like + processes, and finally do the pid file processing (again). + """ + fRc1 = self._executeSubDriver([ 'abort', ], fMaySkip = False, fPreloadASan = True); + fRc2 = self._killAllVBoxProcesses(); + fRc3 = TestDriverBase.actionAbort(self); + return fRc1 and fRc2 and fRc3; + + + # + # Persistent variables. + # + ## @todo integrate into the base driver. Persistent accross scratch wipes? + + def __persistentVarCalcName(self, sVar): + """Returns the (full) filename for the given persistent variable.""" + assert re.match(r'^[a-zA-Z0-9_-]*$', sVar) is not None; + return os.path.join(self.sScratchPath, 'persistent-%s.var' % (sVar,)); + + def _persistentVarSet(self, sVar, sValue = ''): + """ + Sets a persistent variable. + + Returns True on success, False + reporter.error on failure. + + May raise exception if the variable name is invalid or something + unexpected happens. + """ + sFull = self.__persistentVarCalcName(sVar); + try: + with open(sFull, 'w') as oFile: # pylint: disable=unspecified-encoding + if sValue: + oFile.write(sValue.encode('utf-8')); + except: + reporter.errorXcpt('Error creating "%s"' % (sFull,)); + return False; + return True; + + def _persistentVarUnset(self, sVar): + """ + Unsets a persistent variable. + + Returns True on success, False + reporter.error on failure. + + May raise exception if the variable name is invalid or something + unexpected happens. + """ + sFull = self.__persistentVarCalcName(sVar); + if os.path.exists(sFull): + try: + os.unlink(sFull); + except: + reporter.errorXcpt('Error unlinking "%s"' % (sFull,)); + return False; + return True; + + def _persistentVarExists(self, sVar): + """ + Checks if a persistent variable exists. + + Returns true/false. + + May raise exception if the variable name is invalid or something + unexpected happens. + """ + return os.path.exists(self.__persistentVarCalcName(sVar)); + + def _persistentVarGet(self, sVar): + """ + Gets the value of a persistent variable. + + Returns variable value on success. + Returns None if the variable doesn't exist or if an + error (reported) occured. + + May raise exception if the variable name is invalid or something + unexpected happens. + """ + sFull = self.__persistentVarCalcName(sVar); + if not os.path.exists(sFull): + return None; + try: + with open(sFull, 'r') as oFile: # pylint: disable=unspecified-encoding + sValue = oFile.read().decode('utf-8'); + except: + reporter.errorXcpt('Error creating "%s"' % (sFull,)); + return None; + return sValue; + + + # + # Helpers. + # + + def _killAllVBoxProcesses(self): + """ + Kills all virtual box related processes we find in the system. + """ + sHostOs = utils.getHostOs(); + asDebuggers = [ 'cdb', 'windbg', ] if sHostOs == 'windows' else [ 'gdb', 'gdb-i386-apple-darwin', 'lldb' ]; + + for iIteration in range(22): + # Gather processes to kill. + aoTodo = []; + aoDebuggers = []; + for oProcess in utils.processListAll(): + sBase = oProcess.getBaseImageNameNoExeSuff(); + if sBase is None: + continue; + sBase = sBase.lower(); + if sBase in [ 'vboxsvc', 'vboxsds', 'virtualbox', 'virtualboxvm', 'vboxheadless', 'vboxmanage', 'vboxsdl', + 'vboxwebsrv', 'vboxautostart', 'vboxballoonctrl', 'vboxbfe', 'vboxextpackhelperapp', 'vboxnetdhcp', + 'vboxnetnat', 'vboxnetadpctl', 'vboxtestogl', 'vboxtunctl', 'vboxvmmpreload', 'vboxxpcomipcd', ]: + aoTodo.append(oProcess); + if sBase.startswith('virtualbox-') and sBase.endswith('-multiarch.exe'): + aoTodo.append(oProcess); + if sBase in asDebuggers: + aoDebuggers.append(oProcess); + if iIteration in [0, 21]: + reporter.log('Warning: debugger running: %s (%s %s)' % (oProcess.iPid, sBase, oProcess.asArgs)); + if not aoTodo: + return True; + + # Are any of the debugger processes hooked up to a VBox process? + if sHostOs == 'windows': + # On demand debugging windows: windbg -p <decimal-pid> -e <decimal-event> -g + for oDebugger in aoDebuggers: + for oProcess in aoTodo: + # The whole command line is asArgs[0] here. Fix if that changes. + if oDebugger.asArgs and oDebugger.asArgs[0].find('-p %s ' % (oProcess.iPid,)) >= 0: + aoTodo.append(oDebugger); + break; + else: + for oDebugger in aoDebuggers: + for oProcess in aoTodo: + # Simplistic approach: Just check for argument equaling our pid. + if oDebugger.asArgs and ('%s' % oProcess.iPid) in oDebugger.asArgs: + aoTodo.append(oDebugger); + break; + + # Kill. + for oProcess in aoTodo: + reporter.log('Loop #%d - Killing %s (%s, uid=%s)' + % ( iIteration, oProcess.iPid, oProcess.sImage if oProcess.sName is None else oProcess.sName, + oProcess.iUid, )); + if not utils.processKill(oProcess.iPid) \ + and sHostOs != 'windows' \ + and utils.processExists(oProcess.iPid): + # Many of the vbox processes are initially set-uid-to-root and associated debuggers are running + # via sudo, so we might not be able to kill them unless we sudo and use /bin/kill. + try: utils.sudoProcessCall(['/bin/kill', '-9', '%s' % (oProcess.iPid,)]); + except: reporter.logXcpt(); + + # Check if they're all dead like they should be. + time.sleep(0.1); + for oProcess in aoTodo: + if utils.processExists(oProcess.iPid): + time.sleep(2); + break; + + return False; + + def _executeSync(self, asArgs, fMaySkip = False): + """ + Executes a child process synchronously. + + Returns True if the process executed successfully and returned 0. + Returns None if fMaySkip is true and the child exits with RTEXITCODE_SKIPPED. + Returns False for all other cases. + """ + reporter.log('Executing: %s' % (asArgs, )); + reporter.flushall(); + try: + iRc = utils.processCall(asArgs, shell = False, close_fds = False); + except: + reporter.errorXcpt(); + return False; + reporter.log('Exit code: %s (%s)' % (iRc, asArgs)); + if fMaySkip and iRc == rtexitcode.RTEXITCODE_SKIPPED: + return None; + return iRc == 0; + + def _sudoExecuteSync(self, asArgs): + """ + Executes a sudo child process synchronously. + Returns a tuple [True, 0] if the process executed successfully + and returned 0, otherwise [False, rc] is returned. + """ + reporter.log('Executing [sudo]: %s' % (asArgs, )); + reporter.flushall(); + iRc = 0; + try: + iRc = utils.sudoProcessCall(asArgs, shell = False, close_fds = False); + except: + reporter.errorXcpt(); + return (False, 0); + reporter.log('Exit code [sudo]: %s (%s)' % (iRc, asArgs)); + return (iRc == 0, iRc); + + def _findASanLibsForASanBuild(self): + """ + Returns a list of (address) santizier related libraries to preload + when launching the sub driver. + Returns empty list for non-asan builds or on platforms where this isn't needed. + """ + # Note! We include libasan.so.X in the VBoxAll tarball for asan builds, so we + # can use its presence both to detect an 'asan' build and to return it. + # Only the libasan.so.X library needs preloading at present. + if self.sHost in ('linux',): + sLibASan = self._findFile(r'libasan\.so\..*'); + if sLibASan: + return [sLibASan,]; + return []; + + def _executeSubDriver(self, asActions, fMaySkip = True, fPreloadASan = True): + """ + Execute the sub testdriver with the specified action. + """ + asArgs = list(self._asSubDriver) + asArgs.append('--no-wipe-clean'); + asArgs.extend(asActions); + + asASanLibs = []; + if fPreloadASan: + asASanLibs = self._findASanLibsForASanBuild(); + if asASanLibs: + os.environ['LD_PRELOAD'] = ':'.join(asASanLibs); + os.environ['LSAN_OPTIONS'] = 'detect_leaks=0'; # We don't want python leaks. vbox.py disables this. + + # Because of https://github.com/google/sanitizers/issues/856 we must try use setarch to disable + # address space randomization. + + reporter.log('LD_PRELOAD...') + if utils.getHostArch() == 'amd64': + sSetArch = utils.whichProgram('setarch'); + reporter.log('sSetArch=%s' % (sSetArch,)); + if sSetArch: + asArgs = [ sSetArch, 'x86_64', '-R', sys.executable ] + asArgs; + reporter.log('asArgs=%s' % (asArgs,)); + + rc = self._executeSync(asArgs, fMaySkip = fMaySkip); + + del os.environ['LSAN_OPTIONS']; + del os.environ['LD_PRELOAD']; + return rc; + + return self._executeSync(asArgs, fMaySkip = fMaySkip); + + def _maybeUnpackArchive(self, sMaybeArchive, fNonFatal = False): + """ + Attempts to unpack the given build file. + Updates _asBuildFiles. + Returns True/False. No exceptions. + """ + def unpackFilter(sMember): + # type: (string) -> bool + """ Skips debug info. """ + sLower = sMember.lower(); + if sLower.endswith('.pdb'): + return False; + return True; + + asMembers = utils.unpackFile(sMaybeArchive, self.sScratchPath, reporter.log, + reporter.log if fNonFatal else reporter.error, + fnFilter = unpackFilter); + if asMembers is None: + return False; + self._asBuildFiles.extend(asMembers); + return True; + + + def _installVBox(self): + """ + Download / copy the build files into the scratch area and install them. + """ + reporter.testStart('Installing VirtualBox'); + reporter.log('CWD=%s' % (os.getcwd(),)); # curious + + # + # Download the build files. + # + for i, sBuildUrl in enumerate(self._asBuildUrls): + if webutils.downloadFile(sBuildUrl, self._asBuildFiles[i], self.sBuildPath, reporter.log, reporter.log) is not True: + reporter.testDone(fSkipped = True); + return None; # Failed to get binaries, probably deleted. Skip the test run. + + # + # Unpack anything we know what is and append it to the build files + # list. This allows us to use VBoxAll*.tar.gz files. + # + for sFile in list(self._asBuildFiles): # Note! We copy the list as _maybeUnpackArchive updates it. + if self._maybeUnpackArchive(sFile, fNonFatal = True) is not True: + reporter.testDone(fSkipped = True); + return None; # Failed to unpack. Probably local error, like busy + # DLLs on windows, no reason for failing the build. + self._fUnpackedBuildFiles = True; + + # + # Go to system specific installation code. + # + sHost = utils.getHostOs() + if sHost == 'darwin': fRc = self._installVBoxOnDarwin(); + elif sHost == 'linux': fRc = self._installVBoxOnLinux(); + elif sHost == 'solaris': fRc = self._installVBoxOnSolaris(); + elif sHost == 'win': fRc = self._installVBoxOnWindows(); + else: + reporter.error('Unsupported host "%s".' % (sHost,)); + if fRc is False: + reporter.testFailure('Installation error.'); + elif fRc is not True: + reporter.log('Seems installation was skipped. Old version lurking behind? Not the fault of this build/test run!'); + + # + # Install the extension pack. + # + if fRc is True and self._fAutoInstallPuelExtPack: + fRc = self._installExtPack(); + if fRc is False: + reporter.testFailure('Extension pack installation error.'); + + # Some debugging... + try: + cMbFreeSpace = utils.getDiskUsage(self.sScratchPath); + reporter.log('Disk usage after VBox install: %d MB available at %s' % (cMbFreeSpace, self.sScratchPath,)); + except: + reporter.logXcpt('Unable to get disk free space. Ignored. Continuing.'); + + reporter.testDone(fRc is None); + return fRc; + + def _uninstallVBox(self, fIgnoreError = False): + """ + Uninstall VirtualBox. + """ + reporter.testStart('Uninstalling VirtualBox'); + + sHost = utils.getHostOs() + if sHost == 'darwin': fRc = self._uninstallVBoxOnDarwin(); + elif sHost == 'linux': fRc = self._uninstallVBoxOnLinux(); + elif sHost == 'solaris': fRc = self._uninstallVBoxOnSolaris(True); + elif sHost == 'win': fRc = self._uninstallVBoxOnWindows('uninstall'); + else: + reporter.error('Unsupported host "%s".' % (sHost,)); + if fRc is False and not fIgnoreError: + reporter.testFailure('Uninstallation failed.'); + + fRc2 = self._uninstallAllExtPacks(); + if not fRc2 and fRc: + fRc = fRc2; + + reporter.testDone(fSkipped = (fRc is None)); + return fRc; + + def _findFile(self, sRegExp, fMandatory = False): + """ + Returns the first build file that matches the given regular expression + (basename only). + + Returns None if no match was found, logging it as an error if + fMandatory is set. + """ + oRegExp = re.compile(sRegExp); + + reporter.log('_findFile: %s' % (sRegExp,)); + for sFile in self._asBuildFiles: + if oRegExp.match(os.path.basename(sFile)) and os.path.exists(sFile): + return sFile; + + # If we didn't unpack the build files, search all the files in the scratch area: + if not self._fUnpackedBuildFiles: + for sDir, _, asFiles in os.walk(self.sScratchPath): + for sFile in asFiles: + #reporter.log('_findFile: considering %s' % (sFile,)); + if oRegExp.match(sFile): + return os.path.join(sDir, sFile); + + if fMandatory: + reporter.error('Failed to find a file matching "%s" in %s.' % (sRegExp, self._asBuildFiles,)); + return None; + + def _waitForTestManagerConnectivity(self, cSecTimeout): + """ + Check and wait for network connectivity to the test manager. + + This is used with the windows installation and uninstallation since + these usually disrupts network connectivity when installing the filter + driver. If we proceed to quickly, we might finish the test at a time + when we cannot report to the test manager and thus end up with an + abandonded test error. + """ + cSecElapsed = 0; + secStart = utils.timestampSecond(); + while reporter.checkTestManagerConnection() is False: + cSecElapsed = utils.timestampSecond() - secStart; + if cSecElapsed >= cSecTimeout: + reporter.log('_waitForTestManagerConnectivity: Giving up after %u secs.' % (cSecTimeout,)); + return False; + time.sleep(2); + + if cSecElapsed > 0: + reporter.log('_waitForTestManagerConnectivity: Waited %s secs.' % (cSecTimeout,)); + return True; + + + # + # Darwin (Mac OS X). + # + + def _darwinDmgPath(self): + """ Returns the path to the DMG mount.""" + return os.path.join(self.sScratchPath, 'DmgMountPoint'); + + def _darwinUnmountDmg(self, fIgnoreError): + """ + Umount any DMG on at the default mount point. + """ + sMountPath = self._darwinDmgPath(); + if not os.path.exists(sMountPath): + return True; + + # Unmount. + fRc = self._executeSync(['hdiutil', 'detach', sMountPath ]); + if not fRc and not fIgnoreError: + # In case it's busy for some reason or another, just retry after a little delay. + for iTry in range(6): + time.sleep(5); + reporter.error('Retry #%s unmount DMT at %s' % (iTry + 1, sMountPath,)); + fRc = self._executeSync(['hdiutil', 'detach', sMountPath ]); + if fRc: + break; + if not fRc: + reporter.error('Failed to unmount DMG at %s' % (sMountPath,)); + + # Remove dir. + try: + os.rmdir(sMountPath); + except: + if not fIgnoreError: + reporter.errorXcpt('Failed to remove directory %s' % (sMountPath,)); + return fRc; + + def _darwinMountDmg(self, sDmg): + """ + Mount the DMG at the default mount point. + """ + self._darwinUnmountDmg(fIgnoreError = True) + + sMountPath = self._darwinDmgPath(); + if not os.path.exists(sMountPath): + try: + os.mkdir(sMountPath, 0o755); + except: + reporter.logXcpt(); + return False; + + return self._executeSync(['hdiutil', 'attach', '-readonly', '-mount', 'required', '-mountpoint', sMountPath, sDmg, ]); + + def _generateWithoutKextsChoicesXmlOnDarwin(self): + """ + Generates the choices XML when kernel drivers are disabled. + None is returned on failure. + """ + sPath = os.path.join(self.sScratchPath, 'DarwinChoices.xml'); + oFile = utils.openNoInherit(sPath, 'wt'); + oFile.write('<?xml version="1.0" encoding="UTF-8"?>\n' + '<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">\n' + '<plist version="1.0">\n' + '<array>\n' + ' <dict>\n' + ' <key>attributeSetting</key>\n' + ' <integer>0</integer>\n' + ' <key>choiceAttribute</key>\n' + ' <string>selected</string>\n' + ' <key>choiceIdentifier</key>\n' + ' <string>choiceVBoxKEXTs</string>\n' + ' </dict>\n' + '</array>\n' + '</plist>\n'); + oFile.close(); + return sPath; + + def _installVBoxOnDarwin(self): + """ Installs VBox on Mac OS X.""" + + # TEMPORARY HACK - START + # Don't install the kernel drivers on the testboxes with BigSur and later + # Needs a more generic approach but that one needs more effort. + sHostName = socket.getfqdn(); + if sHostName.startswith('testboxmac10') \ + or sHostName.startswith('testboxmac11'): + self._fKernelDrivers = False; + # TEMPORARY HACK - END + + sDmg = self._findFile('^VirtualBox-.*\\.dmg$'); + if sDmg is None: + return False; + + # Mount the DMG. + fRc = self._darwinMountDmg(sDmg); + if fRc is not True: + return False; + + # Uninstall any previous vbox version first. + sUninstaller = os.path.join(self._darwinDmgPath(), 'VirtualBox_Uninstall.tool'); + fRc, _ = self._sudoExecuteSync([sUninstaller, '--unattended',]); + if fRc is True: + + # Install the package. + sPkg = os.path.join(self._darwinDmgPath(), 'VirtualBox.pkg'); + if self._fKernelDrivers: + fRc, _ = self._sudoExecuteSync(['installer', '-verbose', '-dumplog', '-pkg', sPkg, '-target', '/']); + else: + sChoicesXml = self._generateWithoutKextsChoicesXmlOnDarwin(); + if sChoicesXml is not None: + fRc, _ = self._sudoExecuteSync(['installer', '-verbose', '-dumplog', '-pkg', sPkg, \ + '-applyChoiceChangesXML', sChoicesXml, '-target', '/']); + else: + fRc = False; + + # Unmount the DMG and we're done. + if not self._darwinUnmountDmg(fIgnoreError = False): + fRc = False; + return fRc; + + def _uninstallVBoxOnDarwin(self): + """ Uninstalls VBox on Mac OS X.""" + + # Is VirtualBox installed? If not, don't try uninstall it. + sVBox = self._getVBoxInstallPath(fFailIfNotFound = False); + if sVBox is None: + return True; + + # Find the dmg. + sDmg = self._findFile('^VirtualBox-.*\\.dmg$'); + if sDmg is None: + return False; + if not os.path.exists(sDmg): + return True; + + # Mount the DMG. + fRc = self._darwinMountDmg(sDmg); + if fRc is not True: + return False; + + # Execute the uninstaller. + sUninstaller = os.path.join(self._darwinDmgPath(), 'VirtualBox_Uninstall.tool'); + fRc, _ = self._sudoExecuteSync([sUninstaller, '--unattended',]); + + # Unmount the DMG and we're done. + if not self._darwinUnmountDmg(fIgnoreError = False): + fRc = False; + return fRc; + + # + # GNU/Linux + # + + def _installVBoxOnLinux(self): + """ Installs VBox on Linux.""" + sRun = self._findFile('^VirtualBox-.*\\.run$'); + if sRun is None: + return False; + utils.chmodPlusX(sRun); + + # Install the new one. + fRc, _ = self._sudoExecuteSync([sRun,]); + return fRc; + + def _uninstallVBoxOnLinux(self): + """ Uninstalls VBox on Linux.""" + + # Is VirtualBox installed? If not, don't try uninstall it. + sVBox = self._getVBoxInstallPath(fFailIfNotFound = False); + if sVBox is None: + return True; + + # Find the .run file and use it. + sRun = self._findFile('^VirtualBox-.*\\.run$', fMandatory = False); + if sRun is not None: + utils.chmodPlusX(sRun); + fRc, _ = self._sudoExecuteSync([sRun, 'uninstall']); + return fRc; + + # Try the installed uninstaller. + for sUninstaller in [os.path.join(sVBox, 'uninstall.sh'), '/opt/VirtualBox/uninstall.sh', ]: + if os.path.isfile(sUninstaller): + reporter.log('Invoking "%s"...' % (sUninstaller,)); + fRc, _ = self._sudoExecuteSync([sUninstaller, 'uninstall']); + return fRc; + + reporter.log('Did not find any VirtualBox install to uninstall.'); + return True; + + + # + # Solaris + # + + def _generateAutoResponseOnSolaris(self): + """ + Generates an autoresponse file on solaris, returning the name. + None is return on failure. + """ + sPath = os.path.join(self.sScratchPath, 'SolarisAutoResponse'); + oFile = utils.openNoInherit(sPath, 'wt'); + oFile.write('basedir=default\n' + 'runlevel=nocheck\n' + 'conflict=quit\n' + 'setuid=nocheck\n' + 'action=nocheck\n' + 'partial=quit\n' + 'instance=unique\n' + 'idepend=quit\n' + 'rdepend=quit\n' + 'space=quit\n' + 'mail=\n'); + oFile.close(); + return sPath; + + def _installVBoxOnSolaris(self): + """ Installs VBox on Solaris.""" + sPkg = self._findFile('^VirtualBox-.*\\.pkg$', fMandatory = False); + if sPkg is None: + sTar = self._findFile('^VirtualBox-.*-SunOS-.*\\.tar.gz$', fMandatory = False); + if sTar is not None: + if self._maybeUnpackArchive(sTar) is not True: + return False; + sPkg = self._findFile('^VirtualBox-.*\\.pkg$', fMandatory = True); + sRsp = self._findFile('^autoresponse$', fMandatory = True); + if sPkg is None or sRsp is None: + return False; + + # Uninstall first (ignore result). + self._uninstallVBoxOnSolaris(False); + + # Install the new one. + fRc, _ = self._sudoExecuteSync(['pkgadd', '-d', sPkg, '-n', '-a', sRsp, 'SUNWvbox']); + return fRc; + + def _uninstallVBoxOnSolaris(self, fRestartSvcConfigD): + """ Uninstalls VBox on Solaris.""" + reporter.flushall(); + if utils.processCall(['pkginfo', '-q', 'SUNWvbox']) != 0: + return True; + sRsp = self._generateAutoResponseOnSolaris(); + fRc, _ = self._sudoExecuteSync(['pkgrm', '-n', '-a', sRsp, 'SUNWvbox']); + + # + # Restart the svc.configd as it has a tendency to clog up with time and + # become unresponsive. It will handle SIGHUP by exiting the sigwait() + # look in the main function and shut down the service nicely (backend_fini). + # The restarter will then start a new instance of it. + # + if fRestartSvcConfigD: + time.sleep(1); # Give it a chance to flush pkgrm stuff. + self._sudoExecuteSync(['pkill', '-HUP', 'svc.configd']); + time.sleep(5); # Spare a few cpu cycles it to shutdown and restart. + + return fRc; + + # + # Windows + # + + ## VBox windows services we can query the status of. + kasWindowsServices = [ 'vboxsup', 'vboxusbmon', 'vboxnetadp', 'vboxnetflt', 'vboxnetlwf' ]; + + def _installVBoxOnWindows(self): + """ Installs VBox on Windows.""" + sExe = self._findFile('^VirtualBox-.*-(MultiArch|Win).exe$'); + if sExe is None: + return False; + + # TEMPORARY HACK - START + # It seems that running the NDIS cleanup script upon uninstallation is not + # a good idea, so let's run it before installing VirtualBox. + #sHostName = socket.getfqdn(); + #if not sHostName.startswith('testboxwin3') \ + # and not sHostName.startswith('testboxharp2') \ + # and not sHostName.startswith('wei01-b6ka-3') \ + # and utils.getHostOsVersion() in ['8', '8.1', '9', '2008Server', '2008ServerR2', '2012Server']: + # reporter.log('Peforming extra NDIS cleanup...'); + # sMagicScript = os.path.abspath(os.path.join(g_ksValidationKitDir, 'testdriver', 'win-vbox-net-uninstall.ps1')); + # fRc2, _ = self._sudoExecuteSync(['powershell.exe', '-Command', 'set-executionpolicy unrestricted']); + # if not fRc2: + # reporter.log('set-executionpolicy failed.'); + # self._sudoExecuteSync(['powershell.exe', '-Command', 'get-executionpolicy']); + # fRc2, _ = self._sudoExecuteSync(['powershell.exe', '-File', sMagicScript]); + # if not fRc2: + # reporter.log('NDIS cleanup failed.'); + # TEMPORARY HACK - END + + # Uninstall any previous vbox version first. + fRc = self._uninstallVBoxOnWindows('install'); + if fRc is not True: + return None; # There shouldn't be anything to uninstall, and if there is, it's not our fault. + + # Install the MS Visual Studio Redistributable, if requested. (VBox 7.0+ needs this installed once.) + if self._fInstallMsCrt: + reporter.log('Installing MS Visual Studio Redistributable (untested code)...'); + ## @todo Test this. + ## @todo We could cache this on the testrsrc share. + sName = "vc_redist.x64.exe" + sUrl = "https://aka.ms/vs/17/release/" + sName # Permalink, according to MS. + sExe = os.path.join(self.sBuildPath, sName); + if webutils.downloadFile(sUrl, sExe, None, reporter.log, reporter.log): + asArgs = [ sExe, '/Q' ]; + fRc2, iRc = self._sudoExecuteSync(asArgs); + if fRc2 is False: + return reporter.error('Installing MS Visual Studio Redistributable failed, exit code: %s' % (iRc,)); + reporter.log('Installing MS Visual Studio Redistributable done'); + else: + return False; + + # We need the help text to detect supported options below. + reporter.log('Executing: %s' % ([sExe, '--silent', '--help'], )); + reporter.flushall(); + (iExitCode, sHelp, _) = utils.processOutputUnchecked([sExe, '--silent', '--help'], fIgnoreEncoding = True); + reporter.log('Exit code: %d, %u chars of help text' % (iExitCode, len(sHelp),)); + + # Gather installer arguments. + asArgs = [sExe, '-vvvv', '--silent', '--logging']; + asArgs.extend(['--msiparams', 'REBOOT=ReallySuppress']); + sVBoxInstallPath = os.environ.get('VBOX_INSTALL_PATH', None); + if sVBoxInstallPath is not None: + asArgs.extend(['INSTALLDIR="%s"' % (sVBoxInstallPath,)]); + + if sHelp.find("--msi-log-file") >= 0: + sLogFile = os.path.join(self.sScratchPath, 'VBoxInstallLog.txt'); # Specify location to prevent a random one. + asArgs.extend(['--msi-log-file', sLogFile]); + else: + sLogFile = os.path.join(tempfile.gettempdir(), 'VirtualBox', 'VBoxInstallLog.txt'); # Hardcoded TMP location. + + if self._fWinForcedInstallTimestampCA and sHelp.find("--force-install-timestamp-ca") >= 0: + asArgs.extend(['--force-install-timestamp-ca']); + + # Install it. + fRc2, iRc = self._sudoExecuteSync(asArgs); + if fRc2 is False: + if iRc == 3010: # ERROR_SUCCESS_REBOOT_REQUIRED + reporter.error('Installer required a reboot to complete installation (ERROR_SUCCESS_REBOOT_REQUIRED)'); + else: + reporter.error('Installer failed, exit code: %s' % (iRc,)); + fRc = False; + + # Add the installer log if present and wait for the network connection to be restore after the filter driver upset. + if os.path.isfile(sLogFile): + reporter.addLogFile(sLogFile, 'log/installer', "Verbose MSI installation log file"); + self._waitForTestManagerConnectivity(30); + + return fRc; + + def _isProcessPresent(self, sName): + """ Checks whether the named process is present or not. """ + for oProcess in utils.processListAll(): + sBase = oProcess.getBaseImageNameNoExeSuff(); + if sBase is not None and sBase.lower() == sName: + return True; + return False; + + def _killProcessesByName(self, sName, sDesc, fChildren = False): + """ Kills the named process, optionally including children. """ + cKilled = 0; + aoProcesses = utils.processListAll(); + for oProcess in aoProcesses: + sBase = oProcess.getBaseImageNameNoExeSuff(); + if sBase is not None and sBase.lower() == sName: + reporter.log('Killing %s process: %s (%s)' % (sDesc, oProcess.iPid, sBase)); + utils.processKill(oProcess.iPid); + cKilled += 1; + + if fChildren: + for oChild in aoProcesses: + if oChild.iParentPid == oProcess.iPid and oChild.iParentPid is not None: + reporter.log('Killing %s child process: %s (%s)' % (sDesc, oChild.iPid, sBase)); + utils.processKill(oChild.iPid); + cKilled += 1; + return cKilled; + + def _terminateProcessesByNameAndArgSubstr(self, sName, sArg, sDesc): + """ + Terminates the named process using taskkill.exe, if any of its args + contains the passed string. + """ + cKilled = 0; + aoProcesses = utils.processListAll(); + for oProcess in aoProcesses: + sBase = oProcess.getBaseImageNameNoExeSuff(); + if sBase is not None and sBase.lower() == sName and any(sArg in s for s in oProcess.asArgs): + + reporter.log('Killing %s process: %s (%s)' % (sDesc, oProcess.iPid, sBase)); + self._executeSync(['taskkill.exe', '/pid', '%u' % (oProcess.iPid,)]); + cKilled += 1; + return cKilled; + + def _uninstallVBoxOnWindows(self, sMode): + """ + Uninstalls VBox on Windows, all installations we find to be on the safe side... + """ + assert sMode in ['install', 'uninstall',]; + + import win32com.client; # pylint: disable=import-error + win32com.client.gencache.EnsureModule('{000C1092-0000-0000-C000-000000000046}', 1033, 1, 0); + oInstaller = win32com.client.Dispatch('WindowsInstaller.Installer', + resultCLSID = '{000C1090-0000-0000-C000-000000000046}') + + # Search installed products for VirtualBox. + asProdCodes = []; + for sProdCode in oInstaller.Products: + try: + sProdName = oInstaller.ProductInfo(sProdCode, "ProductName"); + except: + reporter.logXcpt(); + continue; + #reporter.log('Info: %s=%s' % (sProdCode, sProdName)); + if sProdName.startswith('Oracle VM VirtualBox') \ + or sProdName.startswith('Sun VirtualBox'): + asProdCodes.append([sProdCode, sProdName]); + + # Before we start uninstalling anything, just ruthlessly kill any cdb, + # msiexec, drvinst and some rundll process we might find hanging around. + if self._isProcessPresent('rundll32'): + cTimes = 0; + while cTimes < 3: + cTimes += 1; + cKilled = self._terminateProcessesByNameAndArgSubstr('rundll32', 'InstallSecurityPromptRunDllW', + 'MSI driver installation'); + if cKilled <= 0: + break; + time.sleep(10); # Give related drvinst process a chance to clean up after we killed the verification dialog. + + if self._isProcessPresent('drvinst'): + time.sleep(15); # In the hope that it goes away. + cTimes = 0; + while cTimes < 4: + cTimes += 1; + cKilled = self._killProcessesByName('drvinst', 'MSI driver installation', True); + if cKilled <= 0: + break; + time.sleep(10); # Give related MSI process a chance to clean up after we killed the driver installer. + + if self._isProcessPresent('msiexec'): + cTimes = 0; + while cTimes < 3: + reporter.log('found running msiexec process, waiting a bit...'); + time.sleep(20) # In the hope that it goes away. + if not self._isProcessPresent('msiexec'): + break; + cTimes += 1; + ## @todo this could also be the msiexec system service, try to detect this case! + if cTimes >= 6: + cKilled = self._killProcessesByName('msiexec', 'MSI driver installation'); + if cKilled > 0: + time.sleep(16); # fudge. + + # cdb.exe sometimes stays running (from utils.getProcessInfo), blocking + # the scratch directory. No idea why. + if self._isProcessPresent('cdb'): + cTimes = 0; + while cTimes < 3: + cKilled = self._killProcessesByName('cdb', 'cdb.exe from getProcessInfo'); + if cKilled <= 0: + break; + time.sleep(2); # fudge. + + # Do the uninstalling. + fRc = True; + sLogFile = os.path.join(self.sScratchPath, 'VBoxUninstallLog.txt'); + for sProdCode, sProdName in asProdCodes: + reporter.log('Uninstalling %s (%s)...' % (sProdName, sProdCode)); + fRc2, iRc = self._sudoExecuteSync(['msiexec', '/uninstall', sProdCode, '/quiet', '/passive', '/norestart', + '/L*v', '%s' % (sLogFile), ]); + if fRc2 is False: + if iRc == 3010: # ERROR_SUCCESS_REBOOT_REQUIRED + reporter.error('Uninstaller required a reboot to complete uninstallation'); + else: + reporter.error('Uninstaller failed, exit code: %s' % (iRc,)); + fRc = False; + + self._waitForTestManagerConnectivity(30); + + # Upload the log on failure. Do it early if the extra cleanups below causes trouble. + if fRc is False and os.path.isfile(sLogFile): + reporter.addLogFile(sLogFile, 'log/uninstaller', "Verbose MSI uninstallation log file"); + sLogFile = None; + + # Log driver service states (should ls \Driver\VBox* and \Device\VBox*). + fHadLeftovers = False; + asLeftovers = []; + for sService in reversed(self.kasWindowsServices): + cTries = 0; + while True: + fRc2, _ = self._sudoExecuteSync(['sc.exe', 'query', sService]); + if not fRc2: + break; + fHadLeftovers = True; + + cTries += 1; + if cTries > 3: + asLeftovers.append(sService,); + break; + + # Get the status output. + try: + sOutput = utils.sudoProcessOutputChecked(['sc.exe', 'query', sService]); + except: + reporter.logXcpt(); + else: + if re.search(r'STATE\s+:\s*1\s*STOPPED', sOutput) is None: + reporter.log('Trying to stop %s...' % (sService,)); + fRc2, _ = self._sudoExecuteSync(['sc.exe', 'stop', sService]); + time.sleep(1); # fudge + + reporter.log('Trying to delete %s...' % (sService,)); + self._sudoExecuteSync(['sc.exe', 'delete', sService]); + + time.sleep(1); # fudge + + if asLeftovers: + reporter.log('Warning! Leftover VBox drivers: %s' % (', '.join(asLeftovers),)); + fRc = False; + + if fHadLeftovers: + self._waitForTestManagerConnectivity(30); + + # Upload the log if we have any leftovers and didn't upload it already. + if sLogFile is not None and (fRc is False or fHadLeftovers) and os.path.isfile(sLogFile): + reporter.addLogFile(sLogFile, 'log/uninstaller', "Verbose MSI uninstallation log file"); + + return fRc; + + + # + # Extension pack. + # + + def _getVBoxInstallPath(self, fFailIfNotFound): + """ Returns the default VBox installation path. """ + sHost = utils.getHostOs(); + if sHost == 'win': + sProgFiles = os.environ.get('ProgramFiles', 'C:\\Program Files'); + asLocs = [ + os.path.join(sProgFiles, 'Oracle', 'VirtualBox'), + os.path.join(sProgFiles, 'OracleVM', 'VirtualBox'), + os.path.join(sProgFiles, 'Sun', 'VirtualBox'), + ]; + elif sHost in ('linux', 'solaris',): + asLocs = [ '/opt/VirtualBox', '/opt/VirtualBox-3.2', '/opt/VirtualBox-3.1', '/opt/VirtualBox-3.0']; + elif sHost == 'darwin': + asLocs = [ '/Applications/VirtualBox.app/Contents/MacOS' ]; + else: + asLocs = [ '/opt/VirtualBox' ]; + if 'VBOX_INSTALL_PATH' in os.environ: + asLocs.insert(0, os.environ.get('VBOX_INSTALL_PATH', None)); + + for sLoc in asLocs: + if os.path.isdir(sLoc): + return sLoc; + if fFailIfNotFound: + reporter.error('Failed to locate VirtualBox installation: %s' % (asLocs,)); + else: + reporter.log2('Failed to locate VirtualBox installation: %s' % (asLocs,)); + return None; + + def _installExtPack(self): + """ Installs the extension pack. """ + sVBox = self._getVBoxInstallPath(fFailIfNotFound = True); + if sVBox is None: + return False; + sExtPackDir = os.path.join(sVBox, 'ExtensionPacks'); + + if self._uninstallAllExtPacks() is not True: + return False; + + sExtPack = self._findFile('Oracle_VM_VirtualBox_Extension_Pack.vbox-extpack'); + if sExtPack is None: + sExtPack = self._findFile('Oracle_VM_VirtualBox_Extension_Pack.*.vbox-extpack'); + if sExtPack is None: + return True; + + sDstDir = os.path.join(sExtPackDir, 'Oracle_VM_VirtualBox_Extension_Pack'); + reporter.log('Installing extension pack "%s" to "%s"...' % (sExtPack, sExtPackDir)); + fRc, _ = self._sudoExecuteSync([ self.getBinTool('vts_tar'), + '--extract', + '--verbose', + '--gzip', + '--file', sExtPack, + '--directory', sDstDir, + '--file-mode-and-mask', '0644', + '--file-mode-or-mask', '0644', + '--dir-mode-and-mask', '0755', + '--dir-mode-or-mask', '0755', + '--owner', '0', + '--group', '0', + ]); + return fRc; + + def _uninstallAllExtPacks(self): + """ Uninstalls all extension packs. """ + sVBox = self._getVBoxInstallPath(fFailIfNotFound = False); + if sVBox is None: + return True; + + sExtPackDir = os.path.join(sVBox, 'ExtensionPacks'); + if not os.path.exists(sExtPackDir): + return True; + + fRc, _ = self._sudoExecuteSync([self.getBinTool('vts_rm'), '-Rfv', '--', sExtPackDir]); + return fRc; + + + +if __name__ == '__main__': + sys.exit(VBoxInstallerTestDriver().main(sys.argv)); diff --git a/src/VBox/ValidationKit/testdriver/vboxtestfileset.py b/src/VBox/ValidationKit/testdriver/vboxtestfileset.py new file mode 100755 index 00000000..96ad0920 --- /dev/null +++ b/src/VBox/ValidationKit/testdriver/vboxtestfileset.py @@ -0,0 +1,149 @@ +# -*- coding: utf-8 -*- +# $Id: vboxtestfileset.py $ +# pylint: disable=too-many-lines + +""" +Test File Set +""" + +__copyright__ = \ +""" +Copyright (C) 2010-2022 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 153224 $" + + +# Standard Python imports. +import os; +import sys; + +# Validation Kit imports. +from common import utils; +from testdriver import reporter; +from testdriver import testfileset; + +# Python 3 hacks: +if sys.version_info[0] >= 3: + xrange = range; # pylint: disable=redefined-builtin,invalid-name + + +class TestFileSet(testfileset.TestFileSet): + """ + A generated set of files and directories for uploading to a VM. + + The file and directory names are compatible with the host, so it is + possible to copy them to the host without changing any names. + + Uploaded as a tarball and expanded via TXS (if new enough) or uploaded vts_tar + utility from the validation kit. + """ + + def __init__(self, oTestVm, sBasePath, sSubDir, # pylint: disable=too-many-arguments + oRngFileSizes = xrange(0, 16384), + oRngManyFiles = xrange(128, 512), + oRngTreeFiles = xrange(128, 384), + oRngTreeDepth = xrange(92, 256), + oRngTreeDirs = xrange(2, 16), + cchMaxPath = 230, + cchMaxName = 230, + asCompatibleWith = None, + uSeed = None): + + asCompOses = [oTestVm.getGuestOs(), ]; + sHostOs = utils.getHostOs(); + if sHostOs not in asCompOses: + asCompOses.append(sHostOs); + + testfileset.TestFileSet.__init__(self, + fDosStyle = oTestVm.isWindows() or oTestVm.isOS2(), + asCompatibleWith = asCompOses, + sBasePath = sBasePath, + sSubDir = sSubDir, + oRngFileSizes = oRngFileSizes, + oRngManyFiles = oRngManyFiles, + oRngTreeFiles = oRngTreeFiles, + oRngTreeDepth = oRngTreeDepth, + oRngTreeDirs = oRngTreeDirs, + cchMaxPath = cchMaxPath, + cchMaxName = cchMaxName, + uSeed = uSeed); + self.oTestVm = oTestVm; + + def __uploadFallback(self, oTxsSession, sTarFileGst, oTstDrv): + """ + Fallback upload method. + """ + sVtsTarExe = 'vts_tar' + self.oTestVm.getGuestExeSuff(); + sVtsTarHst = os.path.join(oTstDrv.sVBoxValidationKit, self.oTestVm.getGuestOs(), + self.oTestVm.getGuestArch(), sVtsTarExe); + sVtsTarGst = self.oTestVm.pathJoin(self.sBasePath, sVtsTarExe); + + if oTxsSession.syncUploadFile(sVtsTarHst, sVtsTarGst) is not True: + return reporter.error('Failed to upload "%s" to the guest as "%s"!' % (sVtsTarHst, sVtsTarGst,)); + + fRc = oTxsSession.syncExec(sVtsTarGst, [sVtsTarGst, '-xzf', sTarFileGst, '-C', self.sBasePath,], fWithTestPipe = False); + if fRc is not True: + return reporter.error('vts_tar failed!'); + return True; + + def upload(self, oTxsSession, oTstDrv): + """ + Uploads the files into the guest via the given TXS session. + + Returns True / False. + """ + + # + # Create a tarball. + # + sTarFileHst = os.path.join(oTstDrv.sScratchPath, 'tdAddGuestCtrl-1-Stuff.tar.gz'); + sTarFileGst = self.oTestVm.pathJoin(self.sBasePath, 'tdAddGuestCtrl-1-Stuff.tar.gz'); + if self.createTarball(sTarFileHst) is not True: + return False; + + # + # Upload it. + # + reporter.log('Uploading tarball "%s" to the guest as "%s"...' % (sTarFileHst, sTarFileGst)); + if oTxsSession.syncUploadFile(sTarFileHst, sTarFileGst) is not True: + return reporter.error('Failed upload tarball "%s" as "%s"!' % (sTarFileHst, sTarFileGst,)); + + # + # Try unpack it. + # + reporter.log('Unpacking "%s" into "%s"...' % (sTarFileGst, self.sBasePath)); + if oTxsSession.syncUnpackFile(sTarFileGst, self.sBasePath, fIgnoreErrors = True) is not True: + reporter.log('Failed to expand tarball "%s" into "%s", falling back on individual directory and file creation...' + % (sTarFileGst, self.sBasePath,)); + if self.__uploadFallback(oTxsSession, sTarFileGst, oTstDrv) is not True: + return False; + reporter.log('Successfully placed test files and directories in the VM.'); + return True; + diff --git a/src/VBox/ValidationKit/testdriver/vboxtestvms.py b/src/VBox/ValidationKit/testdriver/vboxtestvms.py new file mode 100755 index 00000000..8a96f2cf --- /dev/null +++ b/src/VBox/ValidationKit/testdriver/vboxtestvms.py @@ -0,0 +1,2105 @@ +# -*- coding: utf-8 -*- +# $Id: vboxtestvms.py $ + +""" +VirtualBox Test VMs +""" + +__copyright__ = \ +""" +Copyright (C) 2010-2022 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 154728 $" + +# Standard Python imports. +import copy; +import os; +import re; +import random; +import socket; +import string; +import uuid; + +# Validation Kit imports. +from common import pathutils; +from common import utils; +from testdriver import base; +from testdriver import reporter; +from testdriver import vboxcon; + + +# All virtualization modes. +g_asVirtModes = ['hwvirt', 'hwvirt-np', 'raw',]; +# All virtualization modes except for raw-mode. +g_asVirtModesNoRaw = ['hwvirt', 'hwvirt-np',]; +# Dictionary mapping the virtualization mode mnemonics to a little less cryptic +# strings used in test descriptions. +g_dsVirtModeDescs = { + 'raw' : 'Raw-mode', + 'hwvirt' : 'HwVirt', + 'hwvirt-np' : 'NestedPaging' +}; + +## @name VM grouping flags +## @{ +g_kfGrpSmoke = 0x0001; ##< Smoke test VM. +g_kfGrpStandard = 0x0002; ##< Standard test VM. +g_kfGrpStdSmoke = g_kfGrpSmoke | g_kfGrpStandard; ##< shorthand. +g_kfGrpWithGAs = 0x0004; ##< The VM has guest additions installed. +g_kfGrpNoTxs = 0x0008; ##< The VM lacks test execution service. +g_kfGrpAncient = 0x1000; ##< Ancient OS. +g_kfGrpExotic = 0x2000; ##< Exotic OS. +## @} + + +## @name Flags. +## @{ +g_k32 = 32; # pylint: disable=invalid-name +g_k64 = 64; # pylint: disable=invalid-name +g_k32_64 = 96; # pylint: disable=invalid-name +g_kiArchMask = 96; +g_kiNoRaw = 128; ##< No raw mode. +## @} + +# Array indexes. +g_iGuestOsType = 0; +g_iKind = 1; +g_iFlags = 2; +g_iMinCpu = 3; +g_iMaxCpu = 4; +g_iRegEx = 5; + +# Table translating from VM name core to a more detailed guest info. +# pylint: disable=line-too-long +## @todo what's the difference between the first two columns again? +g_aaNameToDetails = \ +[ + [ 'WindowsNT3x', 'WindowsNT3x', g_k32, 1, 32, ['nt3', 'nt3[0-9]*']], # max cpus?? + [ 'WindowsNT4', 'WindowsNT4', g_k32, 1, 32, ['nt4', 'nt4sp[0-9]']], # max cpus?? + [ 'Windows2000', 'Windows2000', g_k32, 1, 32, ['w2k', 'w2ksp[0-9]', 'win2k', 'win2ksp[0-9]']], # max cpus?? + [ 'WindowsXP', 'WindowsXP', g_k32, 1, 32, ['xp', 'xpsp[0-9]']], + [ 'WindowsXP_64', 'WindowsXP_64', g_k64, 1, 32, ['xp64', 'xp64sp[0-9]']], + [ 'Windows2003', 'Windows2003', g_k32, 1, 32, ['w2k3', 'w2k3sp[0-9]', 'win2k3', 'win2k3sp[0-9]']], + [ 'WindowsVista', 'WindowsVista', g_k32, 1, 32, ['vista', 'vistasp[0-9]']], + [ 'WindowsVista_64','WindowsVista_64', g_k64, 1, 64, ['vista-64', 'vistasp[0-9]-64',]], # max cpus/cores?? + [ 'Windows2008', 'Windows2008', g_k32, 1, 64, ['w2k8', 'w2k8sp[0-9]', 'win2k8', 'win2k8sp[0-9]']], # max cpus/cores?? + [ 'Windows2008_64', 'Windows2008_64', g_k64, 1, 64, ['w2k8r2', 'w2k8r2sp[0-9]', 'win2k8r2', 'win2k8r2sp[0-9]']], # max cpus/cores?? + [ 'Windows7', 'Windows7', g_k32, 1, 32, ['w7', 'w7sp[0-9]', 'win7',]], # max cpus/cores?? + [ 'Windows7_64', 'Windows7_64', g_k64, 1, 64, ['w7-64', 'w7sp[0-9]-64', 'win7-64',]], # max cpus/cores?? + [ 'Windows2012', 'Windows2012', g_k64, 1, 64, ['w2k12', 'w2k12sp[0-9]', 'win2k12', 'win2k12sp[0-9]',]], # max cpus/cores?? + [ 'Windows8', 'Windows8', g_k32 | g_kiNoRaw, 1, 32, ['w8', 'w8sp[0-9]', 'win8',]], # max cpus/cores?? + [ 'Windows8_64', 'Windows8_64', g_k64, 1, 64, ['w8-64', 'w8sp[0-9]-64', 'win8-64',]], # max cpus/cores?? + [ 'Windows81', 'Windows81', g_k32 | g_kiNoRaw, 1, 32, ['w81', 'w81sp[0-9]', 'win81',]], # max cpus/cores?? + [ 'Windows81_64', 'Windows81_64', g_k64, 1, 64, ['w81-64', 'w81sp[0-9]-64', 'win81-64',]], # max cpus/cores?? + [ 'Windows10', 'Windows10', g_k32 | g_kiNoRaw, 1, 32, ['w10', 'w10sp[0-9]', 'win10',]], # max cpus/cores?? + [ 'Windows10_64', 'Windows10_64', g_k64, 1, 64, ['w10-64', 'w10sp[0-9]-64', 'win10-64',]], # max cpus/cores?? + [ 'Windows2016', 'Windows2016', g_k64, 1, 64, ['w2k16', 'w2k16sp[0-9]', 'win2k16', 'win2k16sp[0-9]',]], # max cpus/cores?? + [ 'Windows2019', 'Windows2019', g_k64, 1, 64, ['w2k19', 'w2k19sp[0-9]', 'win2k19', 'win2k19sp[0-9]',]], # max cpus/cores?? + [ 'Windows2022', 'Windows2022', g_k64, 1, 64, ['w2k22', 'w2k22sp[0-9]', 'win2k22', 'win2k22sp[0-9]',]], # max cpus/cores?? + [ 'Windows11', 'Windows11', g_k64, 1, 64, ['w11', 'w11-64', 'w11sp[0-9]-64', 'win11', 'win11-64',]], # max cpus/cores?? + [ 'Linux', 'Debian', g_k32, 1, 256, ['deb[0-9]*', 'debian[0-9]*', ]], + [ 'Linux_64', 'Debian_64', g_k64, 1, 256, ['deb[0-9]*-64', 'debian[0-9]*-64', ]], + [ 'Linux', 'RedHat', g_k32, 1, 256, ['rhel', 'rhel[0-9]', 'rhel[0-9]u[0-9]']], + [ 'Linux', 'Fedora', g_k32, 1, 256, ['fedora', 'fedora[0-9]*', ]], + [ 'Linux_64', 'Fedora_64', g_k64, 1, 256, ['fedora-64', 'fedora[0-9]*-64', ]], + [ 'Linux', 'Oracle', g_k32, 1, 256, ['ols[0-9]*', 'oel[0-9]*', ]], + [ 'Linux_64', 'Oracle_64', g_k64, 1, 256, ['ols[0-9]*-64', 'oel[0-9]*-64', ]], + [ 'Linux', 'OpenSUSE', g_k32, 1, 256, ['opensuse[0-9]*', 'suse[0-9]*', ]], + [ 'Linux_64', 'OpenSUSE_64', g_k64, 1, 256, ['opensuse[0-9]*-64', 'suse[0-9]*-64', ]], + [ 'Linux', 'Ubuntu', g_k32, 1, 256, ['ubuntu[0-9]*', ]], + [ 'Linux_64', 'Ubuntu_64', g_k64, 1, 256, ['ubuntu[0-9]*-64', ]], + [ 'Linux', 'ArchLinux', g_k32, 1, 256, ['arch[0-9]*', ]], + [ 'Linux_64', 'ArchLinux_64', g_k64, 1, 256, ['arch[0-9]*-64', ]], + [ 'OS2Warp45', 'OS2Warp45', g_k32 | g_kiNoRaw, 1, 1, ['os2.*', 'acp.*','mcp.*', ]], # smp does busy spinning and unattended installer only does UNI at the momen. + [ 'Solaris', 'Solaris', g_k32, 1, 256, ['sol10', 'sol10u[0-9]']], + [ 'Solaris_64', 'Solaris_64', g_k64, 1, 256, ['sol10-64', 'sol10u-64[0-9]']], + [ 'Solaris_64', 'Solaris11_64', g_k64, 1, 256, ['sol11u1']], + [ 'BSD', 'FreeBSD_64', g_k32_64, 1, 1, ['bs-.*']], # boot sectors, wanted 64-bit type. + [ 'DOS', 'DOS', g_k32, 1, 1, ['bs-.*']], +]; + + +## @name Guest OS type string constants. +## @{ +g_ksGuestOsTypeDarwin = 'darwin'; +g_ksGuestOsTypeDOS = 'dos'; +g_ksGuestOsTypeFreeBSD = 'freebsd'; +g_ksGuestOsTypeLinux = 'linux'; +g_ksGuestOsTypeOS2 = 'os2'; +g_ksGuestOsTypeSolaris = 'solaris'; +g_ksGuestOsTypeWindows = 'windows'; +## @} + +## @name String constants for paravirtualization providers. +## @{ +g_ksParavirtProviderNone = 'none'; +g_ksParavirtProviderDefault = 'default'; +g_ksParavirtProviderLegacy = 'legacy'; +g_ksParavirtProviderMinimal = 'minimal'; +g_ksParavirtProviderHyperV = 'hyperv'; +g_ksParavirtProviderKVM = 'kvm'; +## @} + +## Valid paravirtualization providers. +g_kasParavirtProviders = ( g_ksParavirtProviderNone, g_ksParavirtProviderDefault, g_ksParavirtProviderLegacy, + g_ksParavirtProviderMinimal, g_ksParavirtProviderHyperV, g_ksParavirtProviderKVM ); + +# Mapping for support of paravirtualisation providers per guest OS. +#g_kdaParavirtProvidersSupported = { +# g_ksGuestOsTypeDarwin : ( g_ksParavirtProviderMinimal, ), +# g_ksGuestOsTypeFreeBSD : ( g_ksParavirtProviderNone, g_ksParavirtProviderMinimal, ), +# g_ksGuestOsTypeLinux : ( g_ksParavirtProviderNone, g_ksParavirtProviderMinimal, g_ksParavirtProviderHyperV, g_ksParavirtProviderKVM), +# g_ksGuestOsTypeOS2 : ( g_ksParavirtProviderNone, ), +# g_ksGuestOsTypeSolaris : ( g_ksParavirtProviderNone, ), +# g_ksGuestOsTypeWindows : ( g_ksParavirtProviderNone, g_ksParavirtProviderMinimal, g_ksParavirtProviderHyperV, ) +#} +# Temporary tweak: +# since for the most guests g_ksParavirtProviderNone is almost the same as g_ksParavirtProviderMinimal, +# g_ksParavirtProviderMinimal is removed from the list in order to get maximum number of unique choices +# during independent test runs when paravirt provider is taken randomly. +g_kdaParavirtProvidersSupported = { + g_ksGuestOsTypeDarwin : ( g_ksParavirtProviderMinimal, ), + g_ksGuestOsTypeDOS : ( g_ksParavirtProviderNone, ), + g_ksGuestOsTypeFreeBSD : ( g_ksParavirtProviderNone, ), + g_ksGuestOsTypeLinux : ( g_ksParavirtProviderNone, g_ksParavirtProviderHyperV, g_ksParavirtProviderKVM), + g_ksGuestOsTypeOS2 : ( g_ksParavirtProviderNone, ), + g_ksGuestOsTypeSolaris : ( g_ksParavirtProviderNone, ), + g_ksGuestOsTypeWindows : ( g_ksParavirtProviderNone, g_ksParavirtProviderHyperV, ) +} + + +# pylint: enable=line-too-long + +def _intersects(asSet1, asSet2): + """ + Checks if any of the strings in set 1 matches any of the regular + expressions in set 2. + """ + for sStr1 in asSet1: + for sRx2 in asSet2: + if re.match(sStr1, sRx2 + '$'): + return True; + return False; + + + +class BaseTestVm(object): + """ + Base class for Test VMs. + """ + + def __init__(self, # pylint: disable=too-many-arguments + sVmName, # type: str + fGrouping = 0, # type: int + oSet = None, # type: TestVmSet + sKind = None, # type: str + acCpusSup = None, # type: List[int] + asVirtModesSup = None, # type: List[str] + asParavirtModesSup = None, # type: List[str] + fRandomPvPModeCrap = False, # type: bool + fVmmDevTestingPart = None, # type: bool + fVmmDevTestingMmio = False, # type: bool + iGroup = 1, # type: int + ): + self.oSet = oSet # type: TestVmSet + self.sVmName = sVmName; + self.iGroup = iGroup; # Startup group (for MAC address uniqueness and non-NAT networking). + self.fGrouping = fGrouping; + self.sKind = sKind; # API Guest OS type. + self.acCpusSup = acCpusSup; + self.asVirtModesSup = asVirtModesSup; + self.asParavirtModesSup = asParavirtModesSup; + self.asParavirtModesSupOrg = asParavirtModesSup; # HACK ALERT! Trick to make the 'effing random mess not get in the + # way of actively selecting virtualization modes. + + self.fSkip = False; # All VMs are included in the configured set by default. + self.fSnapshotRestoreCurrent = False; # Whether to restore execution on the current snapshot. + + # VMMDev and serial (TXS++) settings: + self.fVmmDevTestingPart = fVmmDevTestingPart; + self.fVmmDevTestingMmio = fVmmDevTestingMmio; + self.fCom1RawFile = False; + + # Cached stuff (use getters): + self.__sCom1RawFile = None; # Set by createVmInner and getReconfiguredVm if fCom1RawFile is set. + self.__tHddCtrlPortDev = (None, None, None); # The HDD controller, port and device. + self.__tDvdCtrlPortDev = (None, None, None); # The DVD controller, port and device. + self.__cbHdd = -1; # The recommended HDD size. + + # Derived stuff: + self.aInfo = None; + self.sGuestOsType = None; # ksGuestOsTypeXxxx value, API GuestOS Type is in the sKind member. + ## @todo rename sGuestOsType + self._guessStuff(fRandomPvPModeCrap); + + def _mkCanonicalGuestOSType(self, sType): + """ + Convert guest OS type into constant representation. + Raise exception if specified @param sType is unknown. + """ + if sType.lower().startswith('darwin'): + return g_ksGuestOsTypeDarwin + if sType.lower().startswith('bsd'): + return g_ksGuestOsTypeFreeBSD + if sType.lower().startswith('dos'): + return g_ksGuestOsTypeDOS + if sType.lower().startswith('linux'): + return g_ksGuestOsTypeLinux + if sType.lower().startswith('os2'): + return g_ksGuestOsTypeOS2 + if sType.lower().startswith('solaris'): + return g_ksGuestOsTypeSolaris + if sType.lower().startswith('windows'): + return g_ksGuestOsTypeWindows + raise base.GenError(sWhat="unknown guest OS kind: %s" % str(sType)) + + def _guessStuff(self, fRandomPvPModeCrap): + """ + Used by the constructor to guess stuff. + """ + + sNm = self.sVmName.lower().strip(); + asSplit = sNm.replace('-', ' ').split(' '); + + if self.sKind is None: + # From name. + for aInfo in g_aaNameToDetails: + if _intersects(asSplit, aInfo[g_iRegEx]): + self.aInfo = aInfo; + self.sGuestOsType = self._mkCanonicalGuestOSType(aInfo[g_iGuestOsType]) + self.sKind = aInfo[g_iKind]; + break; + if self.sKind is None: + reporter.fatal('The OS of test VM "%s" cannot be guessed' % (self.sVmName,)); + + # Check for 64-bit, if required and supported. + if (self.aInfo[g_iFlags] & g_kiArchMask) == g_k32_64 and _intersects(asSplit, ['64', 'amd64']): + self.sKind = self.sKind + '_64'; + else: + # Lookup the kind. + for aInfo in g_aaNameToDetails: + if self.sKind == aInfo[g_iKind]: + self.aInfo = aInfo; + break; + if self.aInfo is None: + reporter.fatal('The OS of test VM "%s" with sKind="%s" cannot be guessed' % (self.sVmName, self.sKind)); + + # Translate sKind into sGuest OS Type. + if self.sGuestOsType is None: + if self.aInfo is not None: + self.sGuestOsType = self._mkCanonicalGuestOSType(self.aInfo[g_iGuestOsType]) + elif self.sKind.find("Windows") >= 0: + self.sGuestOsType = g_ksGuestOsTypeWindows + elif self.sKind.find("Linux") >= 0: + self.sGuestOsType = g_ksGuestOsTypeLinux; + elif self.sKind.find("Solaris") >= 0: + self.sGuestOsType = g_ksGuestOsTypeSolaris; + elif self.sKind.find("DOS") >= 0: + self.sGuestOsType = g_ksGuestOsTypeDOS; + else: + reporter.fatal('The OS of test VM "%s", sKind="%s" cannot be guessed' % (self.sVmName, self.sKind)); + + # Restrict modes and such depending on the OS. + if self.asVirtModesSup is None: + self.asVirtModesSup = list(g_asVirtModes); + if self.sGuestOsType in (g_ksGuestOsTypeOS2, g_ksGuestOsTypeDarwin) \ + or self.sKind.find('_64') > 0 \ + or (self.aInfo is not None and (self.aInfo[g_iFlags] & g_kiNoRaw)): + self.asVirtModesSup = [sVirtMode for sVirtMode in self.asVirtModesSup if sVirtMode != 'raw']; + # TEMPORARY HACK - START + sHostName = os.environ.get("COMPUTERNAME", None); + if sHostName: sHostName = sHostName.lower(); + else: sHostName = socket.getfqdn(); # Horribly slow on windows without IPv6 DNS/whatever. + if sHostName.startswith('testboxpile1'): + self.asVirtModesSup = [sVirtMode for sVirtMode in self.asVirtModesSup if sVirtMode != 'raw']; + # TEMPORARY HACK - END + + # Restrict the CPU count depending on the OS and/or percieved SMP readiness. + if self.acCpusSup is None: + if _intersects(asSplit, ['uni']): + self.acCpusSup = [1]; + elif self.aInfo is not None: + self.acCpusSup = list(range(self.aInfo[g_iMinCpu], self.aInfo[g_iMaxCpu] + 1)); + else: + self.acCpusSup = [1]; + + # Figure relevant PV modes based on the OS. + if self.asParavirtModesSup is None: + self.asParavirtModesSup = g_kdaParavirtProvidersSupported[self.sGuestOsType]; + ## @todo Remove this hack as soon as we've got around to explictly configure test variations + ## on the server side. Client side random is interesting but not the best option. + self.asParavirtModesSupOrg = self.asParavirtModesSup; + if fRandomPvPModeCrap: + random.seed(); + self.asParavirtModesSup = (random.choice(self.asParavirtModesSup),); + + return True; + + def _generateRawPortFilename(self, oTestDrv, sInfix, sSuffix): + """ Generates a raw port filename. """ + random.seed(); + sRandom = ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(10)); + return os.path.join(oTestDrv.sScratchPath, self.sVmName + sInfix + sRandom + sSuffix); + + def _createVmPre(self, oTestDrv, eNic0AttachType, sDvdImage): + """ + Prepares for creating the VM. + + Returns True / False. + """ + _ = eNic0AttachType; _ = sDvdImage; + if self.fCom1RawFile: + self.__sCom1RawFile = self._generateRawPortFilename(oTestDrv, '-com1-', '.out'); + return True; + + def _createVmDoIt(self, oTestDrv, eNic0AttachType, sDvdImage): + """ + Creates the VM. + + The default implementation creates a VM with defaults, no disks created or attached. + + Returns Wrapped VM object on success, None on failure. + """ + return oTestDrv.createTestVmWithDefaults(self.sVmName, + iGroup = self.iGroup, + sKind = self.sKind, + eNic0AttachType = eNic0AttachType, + sDvdImage = sDvdImage, + fVmmDevTestingPart = self.fVmmDevTestingPart, + fVmmDevTestingMmio = self.fVmmDevTestingMmio, + sCom1RawFile = self.__sCom1RawFile if self.fCom1RawFile else None + ); + + def _createVmPost(self, oTestDrv, oVM, eNic0AttachType, sDvdImage): # type: (base.testdriver, Any, int, str) -> Any + """ + Returns same oVM on success, None on failure (createVm cleans up). + """ + _ = oTestDrv; _ = eNic0AttachType; _ = sDvdImage; + return oVM; + + def _skipVmTest(self, oTestDrv, oVM): + """ + Called by getReconfiguredVm to figure out whether to skip the VM or not. + + Returns True if the VM should be skipped, False otherwise. + """ + _ = oVM; + fHostSupports64bit = oTestDrv.hasHostLongMode(); + if self.is64bitRequired() and not fHostSupports64bit: + reporter.log('Skipping 64-bit VM on non-64 capable host.'); + elif self.isViaIncompatible() and oTestDrv.isHostCpuVia(): + reporter.log('Skipping VIA incompatible VM.'); + elif self.isShanghaiIncompatible() and oTestDrv.isHostCpuShanghai(): + reporter.log('Skipping Shanghai (Zhaoxin) incompatible VM.'); + elif self.isP4Incompatible() and oTestDrv.isHostCpuP4(): + reporter.log('Skipping P4 incompatible VM.'); + else: + return False; + return True; + + + def _childVmReconfig(self, oTestDrv, oVM, oSession): + """ + Hook into getReconfiguredVm() for children. + """ + _ = oTestDrv; _ = oVM; _ = oSession; + return True; + + def _storageCtrlAndBusToName(self, oVBoxMgr, oVM, eCtrl, eBus): + """ + Resolves the storage controller name given type and bus. + + Returns String on success, None on failure w/ errors logged. + """ + try: + aoControllers = oVBoxMgr.getArray(oVM, 'storageControllers'); + except: + reporter.errorXcpt(); + return None; + asSummary = []; + for oController in aoControllers: + try: + eCurCtrl = oController.controllerType; + eCurBus = oController.bus; + sName = oController.name; + except: + reporter.errorXcpt(); + return None; + if eCurCtrl == eCtrl and eCurBus == eBus: + return sName; + asSummary.append('%s-%s-%s' % (eCurCtrl, eCurBus, sName,)); + reporter.error('Unable to find controller of type %s and bus %s (searched: %s)' % (eCtrl, eBus, ', '.join(asSummary),)); + return None; + + + # + # Public interface. + # + + def getResourceSet(self): + """ + Resturns a list of reosurces that the VM needs. + """ + return []; + + def getMissingResources(self, sResourcePath): + """ + Returns a list of missing resources (paths, stuff) that the VM needs. + """ + asRet = []; + asResources = self.getResourceSet(); + for sPath in asResources: + if not os.path.isabs(sPath): + sPath = os.path.join(sResourcePath, sPath); + if not os.path.exists(sPath): + asRet.append(sPath); + return asRet; + + def skipCreatingVm(self, oTestDrv): + """ + Called before VM creation to determine whether the VM should be skipped + due to host incompatibility or something along those lines. + + returns True if it should be skipped, False if not. Caller updates fSkip. + + See also _skipVmTest(). + """ + _ = oTestDrv; + return False; + + + def createVm(self, oTestDrv, eNic0AttachType = None, sDvdImage = None): + """ + Creates the VM with defaults and the few tweaks as per the arguments. + + Returns same as vbox.TestDriver.createTestVM. + """ + reporter.log2(''); + reporter.log2('Creating %s...' % (self.sVmName,)) + oVM = None; + fRc = self._createVmPre(oTestDrv, eNic0AttachType, sDvdImage); + if fRc is True: + oVM = self._createVmDoIt(oTestDrv, eNic0AttachType, sDvdImage); + if oVM: + oVM = self._createVmPost(oTestDrv, oVM, eNic0AttachType, sDvdImage); + return oVM; + + def getReconfiguredVm(self, oTestDrv, cCpus, sVirtMode, sParavirtMode = None): + """ + actionExecute worker that finds and reconfigure a test VM. + + Returns (fRc, oVM) where fRc is True, None or False and oVM is a + VBox VM object that is only present when rc is True. + """ + + fRc = False; + oVM = oTestDrv.getVmByName(self.sVmName); + if oVM is not None: + if self.fSnapshotRestoreCurrent is True: + fRc = True; + else: + fHostSupports64bit = oTestDrv.hasHostLongMode(); + if self._skipVmTest(oTestDrv, oVM): + fRc = None; # Skip the test. + else: + oSession = oTestDrv.openSession(oVM); + if oSession is not None: + fRc = oSession.enableVirtEx(sVirtMode != 'raw'); + fRc = fRc and oSession.enableNestedPaging(sVirtMode == 'hwvirt-np'); + fRc = fRc and oSession.setCpuCount(cCpus); + if cCpus > 1: + fRc = fRc and oSession.enableIoApic(True); + + if sParavirtMode is not None and oSession.fpApiVer >= 5.0: + adParavirtProviders = { + g_ksParavirtProviderNone : vboxcon.ParavirtProvider_None, + g_ksParavirtProviderDefault: vboxcon.ParavirtProvider_Default, + g_ksParavirtProviderLegacy : vboxcon.ParavirtProvider_Legacy, + g_ksParavirtProviderMinimal: vboxcon.ParavirtProvider_Minimal, + g_ksParavirtProviderHyperV : vboxcon.ParavirtProvider_HyperV, + g_ksParavirtProviderKVM : vboxcon.ParavirtProvider_KVM, + }; + fRc = fRc and oSession.setParavirtProvider(adParavirtProviders[sParavirtMode]); + + fCfg64Bit = self.is64bitRequired() or (self.is64bit() and fHostSupports64bit and sVirtMode != 'raw'); + fRc = fRc and oSession.enableLongMode(fCfg64Bit); + if fCfg64Bit: # This is to avoid GUI pedantic warnings in the GUI. Sigh. + oOsType = oSession.getOsType(); + if oOsType is not None: + if oOsType.is64Bit and sVirtMode == 'raw': + assert(oOsType.id[-3:] == '_64'); + fRc = fRc and oSession.setOsType(oOsType.id[:-3]); + elif not oOsType.is64Bit and sVirtMode != 'raw': + fRc = fRc and oSession.setOsType(oOsType.id + '_64'); + + # New serial raw file. + if fRc and self.fCom1RawFile: + self.__sCom1RawFile = self._generateRawPortFilename(oTestDrv, '-com1-', '.out'); + utils.noxcptDeleteFile(self.__sCom1RawFile); + fRc = oSession.setupSerialToRawFile(0, self.__sCom1RawFile); + + # Make life simpler for child classes. + if fRc: + fRc = self._childVmReconfig(oTestDrv, oVM, oSession); + + fRc = fRc and oSession.saveSettings(); + if not oSession.close(): + fRc = False; + if fRc is True: + return (True, oVM); + return (fRc, None); + + def getNonCanonicalGuestOsType(self): + """ + Gets the non-canonical OS type (self.sGuestOsType is canonical). + """ + return self.sKind; #self.aInfo[g_iGuestOsType]; + + def getGuestArch(self): + """ Same as util.getHostArch. """ + return 'amd64' if self.sKind.find('_64') >= 0 else 'x86'; + + def getGuestOs(self): + """ Same as util.getHostOs. """ + if self.isWindows(): return 'win'; + if self.isOS2(): return 'os2'; + if self.isLinux(): return 'linux'; + reporter.error('getGuestOs does not what to return!'); + raise Exception(); + + def getGuestOsDotArch(self): + """ Same as util.getHostOsDotArch. """ + return self.getGuestOs() + '.' + self.getGuestArch(); + + def getGuestExeSuff(self): + """ The executable image suffix for the guest. """ + if self.isWindows() or self.isOS2(): + return '.exe'; + return ''; + + def isWindows(self): + """ Checks if it's a Windows VM. """ + return self.sGuestOsType == g_ksGuestOsTypeWindows; + + def isOS2(self): + """ Checks if it's an OS/2 VM. """ + return self.sGuestOsType == g_ksGuestOsTypeOS2; + + def isLinux(self): + """ Checks if it's an Linux VM. """ + return self.sGuestOsType == g_ksGuestOsTypeLinux; + + def is64bit(self): + """ Checks if it's a 64-bit VM. """ + return self.sKind.find('_64') >= 0; + + def is64bitRequired(self): + """ Check if 64-bit is required or not. """ + return (self.aInfo[g_iFlags] & g_k64) != 0; + + def isLoggedOntoDesktop(self): + """ Checks if the test VM is logging onto a graphical desktop by default. """ + if self.isWindows(): + return True; + if self.isOS2(): + return True; + if self.sVmName.find('-desktop'): + return True; + return False; + + def isViaIncompatible(self): + """ + Identifies VMs that doesn't work on VIA. + + Returns True if NOT supported on VIA, False if it IS supported. + """ + # Oracle linux doesn't like VIA in our experience + if self.aInfo[g_iKind] in ['Oracle', 'Oracle_64']: + return True; + # OS/2: "The system detected an internal processing error at location + # 0168:fff1da1f - 000e:ca1f. 0a8606fd + if self.isOS2(): + return True; + # Windows NT4 before SP4 won't work because of cmpxchg8b not being + # detected, leading to a STOP 3e(80,0,0,0). + if self.aInfo[g_iKind] == 'WindowsNT4': + if self.sVmName.find('sp') < 0: + return True; # no service pack. + if self.sVmName.find('sp0') >= 0 \ + or self.sVmName.find('sp1') >= 0 \ + or self.sVmName.find('sp2') >= 0 \ + or self.sVmName.find('sp3') >= 0: + return True; + # XP x64 on a physical VIA box hangs exactly like a VM. + if self.aInfo[g_iKind] in ['WindowsXP_64', 'Windows2003_64']: + return True; + # Vista 64 throws BSOD 0x5D (UNSUPPORTED_PROCESSOR) + if self.aInfo[g_iKind] in ['WindowsVista_64']: + return True; + # Solaris 11 hangs on VIA, tested on a physical box (testboxvqc) + if self.aInfo[g_iKind] in ['Solaris11_64']: + return True; + return False; + + def isShanghaiIncompatible(self): + """ + Identifies VMs that doesn't work on Shanghai. + + Returns True if NOT supported on Shanghai, False if it IS supported. + """ + # For now treat it just like VIA, to be adjusted later + return self.isViaIncompatible() + + def isP4Incompatible(self): + """ + Identifies VMs that doesn't work on Pentium 4 / Pentium D. + + Returns True if NOT supported on P4, False if it IS supported. + """ + # Stupid 1 kHz timer. Too much for antique CPUs. + if self.sVmName.find('rhel5') >= 0: + return True; + # Due to the boot animation the VM takes forever to boot. + if self.aInfo[g_iKind] == 'Windows2000': + return True; + return False; + + def isHostCpuAffectedByUbuntuNewAmdBug(self, oTestDrv): + """ + Checks if the host OS is affected by older ubuntu installers being very + picky about which families of AMD CPUs it would run on. + + The installer checks for family 15, later 16, later 20, and in 11.10 + they remove the family check for AMD CPUs. + """ + if not oTestDrv.isHostCpuAmd(): + return False; + try: + (uMaxExt, _, _, _) = oTestDrv.oVBox.host.getProcessorCPUIDLeaf(0, 0x80000000, 0); + (uFamilyModel, _, _, _) = oTestDrv.oVBox.host.getProcessorCPUIDLeaf(0, 0x80000001, 0); + except: + reporter.logXcpt(); + return False; + if uMaxExt < 0x80000001 or uMaxExt > 0x8000ffff: + return False; + + uFamily = (uFamilyModel >> 8) & 0xf + if uFamily == 0xf: + uFamily = ((uFamilyModel >> 20) & 0x7f) + 0xf; + ## @todo Break this down into which old ubuntu release supports exactly + ## which AMD family, if we care. + if uFamily <= 15: + return False; + reporter.log('Skipping "%s" because host CPU is a family %u AMD, which may cause trouble for the guest OS installer.' + % (self.sVmName, uFamily,)); + return True; + + def getTestUser(self): + """ + Gets the primary test user name. + """ + if self.isWindows(): + return 'Administrator'; + return 'vbox'; + + def getTestUserPassword(self, sUser = None): + """ + Gets the password for the primary user (or other specified one). + """ + if sUser == 'test': + return ''; + if sUser == 'vboxuser': # Default unattended installation user and password. + return 'changeme'; + return 'password'; + + def getCom1RawFile(self, oVM): + """ + Gets the name of the COM1 raw file. + + Returns string, None on failure or if not active. + + Note! Do not access __sCom1RawFile directly as it will not be set unless the + 'config' action was executed in the same run. + """ + if self.fCom1RawFile: + # Retrieve it from the IMachine object and cache the result if needed: + if self.__sCom1RawFile is None: + try: + oPort = oVM.machine.getSerialPort(0); + except: + reporter.errorXcpt('failed to get serial port #0'); + else: + try: + self.__sCom1RawFile = oPort.path; + except: + reporter.errorXcpt('failed to get the "path" property on serial port #0'); + return self.__sCom1RawFile; + + reporter.error('getCom1RawFile called when fCom1RawFile is False'); + return None; + + def getIGuestOSType(self, oVBoxWrapped): + """ + Gets the IGuestOSType object corresponding to self.sKind. + + Returns object on success, None on failure (logged as error). + """ + try: + return oVBoxWrapped.o.getGuestOSType(self.sKind); + except: + reporter.errorXcpt('sVmName=%s sKind=%s' % (self.sVmName, self.sKind,)); + return None; + + def getRecommendedHddSize(self, oVBoxWrapped): + """ + Gets the recommended HDD size from the IGuestOSType matching self.sKind. + + Returns size in bytes on success, -1 on failure. + """ + if self.__cbHdd < 0: + oGuestOSType = self.getIGuestOSType(oVBoxWrapped); + if oGuestOSType: + try: + self.__cbHdd = oGuestOSType.recommendedHDD; + except: + reporter.errorXcpt(); + return -1; + return self.__cbHdd; + + def getHddAddress(self, oVM, oVBoxWrapped): + """ + Gets the HDD attachment address. + + Returns (sController, iPort, iDevice) on success; (None, None, None) on failure. + + Note! Do not access the cached value directly! + """ + # Cached already? + if self.__tHddCtrlPortDev[0] is not None: + return self.__tHddCtrlPortDev; + + # First look for HDs attached to the VM: + try: + aoAttachments = oVBoxWrapped.oVBoxMgr.getArray(oVM, 'mediumAttachments') + except: + reporter.errorXcpt(); + else: + for oAtt in aoAttachments: + try: + sCtrl = oAtt.controller + iPort = oAtt.port; + iDev = oAtt.device; + eType = oAtt.type; + except: + reporter.errorXcpt(); + return self.__tHddCtrlPortDev; + if eType == vboxcon.DeviceType_HardDisk: + self.__tHddCtrlPortDev = (sCtrl, iPort, iDev); + reporter.log2('getHddAddress: %s, %s, %s' % self.__tHddCtrlPortDev); + return self.__tHddCtrlPortDev; + + # Then consult IGuestOSType: + oGuestOSType = self.getIGuestOSType(oVBoxWrapped); + if oGuestOSType: + try: + eCtrl = oGuestOSType.recommendedHDStorageController; + eBus = oGuestOSType.recommendedHDStorageBus; + except: + reporter.errorXcpt(); + else: + # ASSUMES port 0, device 0. + self.__tHddCtrlPortDev = (self._storageCtrlAndBusToName(oVBoxWrapped.oVBoxMgr, oVM, eCtrl, eBus), 0, 0); + reporter.log2('getHddAddress: %s, %s, %s [IGuestOSType]' % self.__tHddCtrlPortDev); + return self.__tHddCtrlPortDev; + + def getDvdAddress(self, oVM, oVBoxWrapped): + """ + Gets the DVD attachment address. + + Returns (sController, iPort, iDevice) on success; (None, None, None) on failure. + + Note! Do not access the cached value directly! + """ + # Cached already? + if self.__tDvdCtrlPortDev[0] is not None: + return self.__tDvdCtrlPortDev; + + # First look for DVD attached to the VM: + try: + aoAttachments = oVBoxWrapped.oVBoxMgr.getArray(oVM, 'mediumAttachments') + except: + reporter.errorXcpt(); + else: + for oAtt in aoAttachments: + try: + sCtrl = oAtt.controller + iPort = oAtt.port; + iDev = oAtt.device; + eType = oAtt.type; + except: + reporter.errorXcpt(); + return self.__tDvdCtrlPortDev; + if eType == vboxcon.DeviceType_DVD: + self.__tDvdCtrlPortDev = (sCtrl, iPort, iDev); + reporter.log2('getDvdAddress: %s, %s, %s' % self.__tDvdCtrlPortDev); + return self.__tDvdCtrlPortDev; + + # Then consult IGuestOSType: + oGuestOSType = self.getIGuestOSType(oVBoxWrapped); + if oGuestOSType: + try: + eCtrl = oGuestOSType.recommendedDVDStorageController; + eBus = oGuestOSType.recommendedDVDStorageBus; + except: + reporter.errorXcpt(); + else: + # ASSUMES port 1, device 0. + self.__tDvdCtrlPortDev = (self._storageCtrlAndBusToName(oVBoxWrapped.oVBoxMgr, oVM, eCtrl, eBus), 1, 0); + reporter.log2('getDvdAddress: %s, %s, %s [IGuestOSType]' % self.__tDvdCtrlPortDev); + return self.__tDvdCtrlPortDev; + + def recreateRecommendedHdd(self, oVM, oTestDrv, sHddPath = None): + """ + Detaches and delete any current hard disk and then ensures that a new + one with the recommended size is created and attached to the recommended + controller/port/device. + + Returns True/False (errors logged). + """ + # Generate a name if none was given: + if not sHddPath: + try: + sHddPath = oVM.settingsFilePath; + except: + return reporter.errorXcpt(); + sHddPath = os.path.join(os.path.dirname(sHddPath), '%s-%s.vdi' % (self.sVmName, uuid.uuid4(),)); + + fRc = False; + + # Get the hard disk specs first: + cbHdd = self.getRecommendedHddSize(oTestDrv.oVBox); + tHddAddress = self.getHddAddress(oVM, oTestDrv.oVBox); + assert len(tHddAddress) == 3; + if tHddAddress[0] and cbHdd > 0: + # Open an session so we can make changes: + oSession = oTestDrv.openSession(oVM); + if oSession is not None: + # Detach the old disk (this will succeed with oOldHd set to None the first time around). + (fRc, oOldHd) = oSession.detachHd(tHddAddress[0], tHddAddress[1], tHddAddress[2]); + if fRc: + # Create a new disk and attach it. + fRc = oSession.createAndAttachHd(sHddPath, + cb = cbHdd, + sController = tHddAddress[0], + iPort = tHddAddress[1], + iDevice = tHddAddress[2], + fImmutable = False); + if fRc: + # Save the changes. + fRc = oSession.saveSettings(); + + # Delete the old HD: + if fRc and oOldHd is not None: + fRc = fRc and oTestDrv.oVBox.deleteHdByMedium(oOldHd); + fRc = fRc and oSession.saveSettings(); # Necessary for media reg?? + else: + oSession.discardSettings(); + fRc = oSession.close() and fRc; + return fRc; + + def pathJoin(self, sBase, *asAppend): + """ See common.pathutils.joinEx(). """ + return pathutils.joinEx(self.isWindows() or self.isOS2(), sBase, *asAppend); + + def pathSep(self): + """ Returns the preferred paths separator for the guest OS. """ + return '\\' if self.isWindows() or self.isOS2() else '/'; + + +## @todo Inherit from BaseTestVm +class TestVm(object): + """ + A Test VM - name + VDI/whatever. + + This is just a data object. + """ + + def __init__(self, # pylint: disable=too-many-arguments + sVmName, # type: str + fGrouping = 0, # type: int + oSet = None, # type: TestVmSet + sHd = None, # type: str + sKind = None, # type: str + acCpusSup = None, # type: List[int] + asVirtModesSup = None, # type: List[str] + fIoApic = None, # type: bool + fNstHwVirt = False, # type: bool + fPae = None, # type: bool + sNic0AttachType = None, # type: str + sFloppy = None, # type: str + fVmmDevTestingPart = None, # type: bool + fVmmDevTestingMmio = False, # type: bool + asParavirtModesSup = None, # type: List[str] + fRandomPvPMode = False, # type: bool + sFirmwareType = 'bios', # type: str + sChipsetType = 'piix3', # type: str + sIommuType = 'none', # type: str + sHddControllerType = 'IDE Controller', # type: str + sDvdControllerType = 'IDE Controller' # type: str + ): + self.oSet = oSet; + self.sVmName = sVmName; + self.fGrouping = fGrouping; + self.sHd = sHd; # Relative to the testrsrc root. + self.acCpusSup = acCpusSup; + self.asVirtModesSup = asVirtModesSup; + self.asParavirtModesSup = asParavirtModesSup; + self.asParavirtModesSupOrg = asParavirtModesSup; # HACK ALERT! Trick to make the 'effing random mess not get in the + # way of actively selecting virtualization modes. + self.sKind = sKind; + self.sGuestOsType = None; + self.sDvdImage = None; # Relative to the testrsrc root. + self.sDvdControllerType = sDvdControllerType; + self.fIoApic = fIoApic; + self.fNstHwVirt = fNstHwVirt; + self.fPae = fPae; + self.sNic0AttachType = sNic0AttachType; + self.sHddControllerType = sHddControllerType; + self.sFloppy = sFloppy; # Relative to the testrsrc root, except when it isn't... + self.fVmmDevTestingPart = fVmmDevTestingPart; + self.fVmmDevTestingMmio = fVmmDevTestingMmio; + self.sFirmwareType = sFirmwareType; + self.sChipsetType = sChipsetType; + self.sIommuType = sIommuType; + self.fCom1RawFile = False; + + self.fSnapshotRestoreCurrent = False; # Whether to restore execution on the current snapshot. + self.fSkip = False; # All VMs are included in the configured set by default. + self.aInfo = None; + self.sCom1RawFile = None; # Set by createVmInner and getReconfiguredVm if fCom1RawFile is set. + self._guessStuff(fRandomPvPMode); + + def _mkCanonicalGuestOSType(self, sType): + """ + Convert guest OS type into constant representation. + Raise exception if specified @param sType is unknown. + """ + if sType.lower().startswith('darwin'): + return g_ksGuestOsTypeDarwin + if sType.lower().startswith('bsd'): + return g_ksGuestOsTypeFreeBSD + if sType.lower().startswith('dos'): + return g_ksGuestOsTypeDOS + if sType.lower().startswith('linux'): + return g_ksGuestOsTypeLinux + if sType.lower().startswith('os2'): + return g_ksGuestOsTypeOS2 + if sType.lower().startswith('solaris'): + return g_ksGuestOsTypeSolaris + if sType.lower().startswith('windows'): + return g_ksGuestOsTypeWindows + raise base.GenError(sWhat="unknown guest OS kind: %s" % str(sType)) + + def _guessStuff(self, fRandomPvPMode): + """ + Used by the constructor to guess stuff. + """ + + sNm = self.sVmName.lower().strip(); + asSplit = sNm.replace('-', ' ').split(' '); + + if self.sKind is None: + # From name. + for aInfo in g_aaNameToDetails: + if _intersects(asSplit, aInfo[g_iRegEx]): + self.aInfo = aInfo; + self.sGuestOsType = self._mkCanonicalGuestOSType(aInfo[g_iGuestOsType]) + self.sKind = aInfo[g_iKind]; + break; + if self.sKind is None: + reporter.fatal('The OS of test VM "%s" cannot be guessed' % (self.sVmName,)); + + # Check for 64-bit, if required and supported. + if (self.aInfo[g_iFlags] & g_kiArchMask) == g_k32_64 and _intersects(asSplit, ['64', 'amd64']): + self.sKind = self.sKind + '_64'; + else: + # Lookup the kind. + for aInfo in g_aaNameToDetails: + if self.sKind == aInfo[g_iKind]: + self.aInfo = aInfo; + break; + if self.aInfo is None: + reporter.fatal('The OS of test VM "%s" with sKind="%s" cannot be guessed' % (self.sVmName, self.sKind)); + + # Translate sKind into sGuest OS Type. + if self.sGuestOsType is None: + if self.aInfo is not None: + self.sGuestOsType = self._mkCanonicalGuestOSType(self.aInfo[g_iGuestOsType]) + elif self.sKind.find("Windows") >= 0: + self.sGuestOsType = g_ksGuestOsTypeWindows + elif self.sKind.find("Linux") >= 0: + self.sGuestOsType = g_ksGuestOsTypeLinux; + elif self.sKind.find("Solaris") >= 0: + self.sGuestOsType = g_ksGuestOsTypeSolaris; + elif self.sKind.find("DOS") >= 0: + self.sGuestOsType = g_ksGuestOsTypeDOS; + else: + reporter.fatal('The OS of test VM "%s", sKind="%s" cannot be guessed' % (self.sVmName, self.sKind)); + + # Restrict modes and such depending on the OS. + if self.asVirtModesSup is None: + self.asVirtModesSup = list(g_asVirtModes); + if self.sGuestOsType in (g_ksGuestOsTypeOS2, g_ksGuestOsTypeDarwin) \ + or self.sKind.find('_64') > 0 \ + or (self.aInfo is not None and (self.aInfo[g_iFlags] & g_kiNoRaw)): + self.asVirtModesSup = [sVirtMode for sVirtMode in self.asVirtModesSup if sVirtMode != 'raw']; + # TEMPORARY HACK - START + sHostName = os.environ.get("COMPUTERNAME", None); + if sHostName: sHostName = sHostName.lower(); + else: sHostName = socket.getfqdn(); # Horribly slow on windows without IPv6 DNS/whatever. + if sHostName.startswith('testboxpile1'): + self.asVirtModesSup = [sVirtMode for sVirtMode in self.asVirtModesSup if sVirtMode != 'raw']; + # TEMPORARY HACK - END + + # Restrict the CPU count depending on the OS and/or percieved SMP readiness. + if self.acCpusSup is None: + if _intersects(asSplit, ['uni']): + self.acCpusSup = [1]; + elif self.aInfo is not None: + self.acCpusSup = list(range(self.aInfo[g_iMinCpu], self.aInfo[g_iMaxCpu] + 1)); + else: + self.acCpusSup = [1]; + + # Figure relevant PV modes based on the OS. + if self.asParavirtModesSup is None: + self.asParavirtModesSup = g_kdaParavirtProvidersSupported[self.sGuestOsType]; + ## @todo Remove this hack as soon as we've got around to explictly configure test variations + ## on the server side. Client side random is interesting but not the best option. + self.asParavirtModesSupOrg = self.asParavirtModesSup; + if fRandomPvPMode: + random.seed(); + self.asParavirtModesSup = (random.choice(self.asParavirtModesSup),); + + return True; + + def getNonCanonicalGuestOsType(self): + """ + Gets the non-canonical OS type (self.sGuestOsType is canonical). + """ + return self.aInfo[g_iGuestOsType]; + + def getMissingResources(self, sTestRsrc): + """ + Returns a list of missing resources (paths, stuff) that the VM needs. + """ + asRet = []; + for sPath in [ self.sHd, self.sDvdImage, self.sFloppy]: + if sPath is not None: + if not os.path.isabs(sPath): + sPath = os.path.join(sTestRsrc, sPath); + if not os.path.exists(sPath): + asRet.append(sPath); + return asRet; + + def skipCreatingVm(self, oTestDrv): + """ + Called before VM creation to determine whether the VM should be skipped + due to host incompatibility or something along those lines. + + returns True if it should be skipped, False if not. + """ + if self.fNstHwVirt and not oTestDrv.hasHostNestedHwVirt(): + reporter.log('Ignoring VM %s (Nested hardware-virtualization not support on this host).' % (self.sVmName,)); + return True; + return False; + + def createVm(self, oTestDrv, eNic0AttachType = None, sDvdImage = None): + """ + Creates the VM with defaults and the few tweaks as per the arguments. + + Returns same as vbox.TestDriver.createTestVM. + """ + if sDvdImage is not None: + sMyDvdImage = sDvdImage; + else: + sMyDvdImage = self.sDvdImage; + + if eNic0AttachType is not None: + eMyNic0AttachType = eNic0AttachType; + elif self.sNic0AttachType is None: + eMyNic0AttachType = None; + elif self.sNic0AttachType == 'nat': + eMyNic0AttachType = vboxcon.NetworkAttachmentType_NAT; + elif self.sNic0AttachType == 'bridged': + eMyNic0AttachType = vboxcon.NetworkAttachmentType_Bridged; + else: + assert False, self.sNic0AttachType; + + return self.createVmInner(oTestDrv, eMyNic0AttachType, sMyDvdImage); + + def _generateRawPortFilename(self, oTestDrv, sInfix, sSuffix): + """ Generates a raw port filename. """ + random.seed(); + sRandom = ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(10)); + return os.path.join(oTestDrv.sScratchPath, self.sVmName + sInfix + sRandom + sSuffix); + + def createVmInner(self, oTestDrv, eNic0AttachType, sDvdImage): + """ + Same as createVm but parameters resolved. + + Returns same as vbox.TestDriver.createTestVM. + """ + reporter.log2(''); + reporter.log2('Calling createTestVM on %s...' % (self.sVmName,)) + if self.fCom1RawFile: + self.sCom1RawFile = self._generateRawPortFilename(oTestDrv, '-com1-', '.out'); + return oTestDrv.createTestVM(self.sVmName, + 1, # iGroup + sHd = self.sHd, + sKind = self.sKind, + fIoApic = self.fIoApic, + fNstHwVirt = self.fNstHwVirt, + fPae = self.fPae, + eNic0AttachType = eNic0AttachType, + sDvdImage = sDvdImage, + sDvdControllerType = self.sDvdControllerType, + sHddControllerType = self.sHddControllerType, + sFloppy = self.sFloppy, + fVmmDevTestingPart = self.fVmmDevTestingPart, + fVmmDevTestingMmio = self.fVmmDevTestingMmio, + sFirmwareType = self.sFirmwareType, + sChipsetType = self.sChipsetType, + sIommuType = self.sIommuType, + sCom1RawFile = self.sCom1RawFile if self.fCom1RawFile else None + ); + + def getReconfiguredVm(self, oTestDrv, cCpus, sVirtMode, sParavirtMode = None): + """ + actionExecute worker that finds and reconfigure a test VM. + + Returns (fRc, oVM) where fRc is True, None or False and oVM is a + VBox VM object that is only present when rc is True. + """ + + fRc = False; + oVM = oTestDrv.getVmByName(self.sVmName); + if oVM is not None: + if self.fSnapshotRestoreCurrent is True: + fRc = True; + else: + fHostSupports64bit = oTestDrv.hasHostLongMode(); + if self.is64bitRequired() and not fHostSupports64bit: + fRc = None; # Skip the test. + elif self.isViaIncompatible() and oTestDrv.isHostCpuVia(): + fRc = None; # Skip the test. + elif self.isShanghaiIncompatible() and oTestDrv.isHostCpuShanghai(): + fRc = None; # Skip the test. + elif self.isP4Incompatible() and oTestDrv.isHostCpuP4(): + fRc = None; # Skip the test. + else: + oSession = oTestDrv.openSession(oVM); + if oSession is not None: + fRc = oSession.enableVirtEx(sVirtMode != 'raw'); + fRc = fRc and oSession.enableNestedPaging(sVirtMode == 'hwvirt-np'); + fRc = fRc and oSession.setCpuCount(cCpus); + if cCpus > 1: + fRc = fRc and oSession.enableIoApic(True); + + if sParavirtMode is not None and oSession.fpApiVer >= 5.0: + adParavirtProviders = { + g_ksParavirtProviderNone : vboxcon.ParavirtProvider_None, + g_ksParavirtProviderDefault: vboxcon.ParavirtProvider_Default, + g_ksParavirtProviderLegacy : vboxcon.ParavirtProvider_Legacy, + g_ksParavirtProviderMinimal: vboxcon.ParavirtProvider_Minimal, + g_ksParavirtProviderHyperV : vboxcon.ParavirtProvider_HyperV, + g_ksParavirtProviderKVM : vboxcon.ParavirtProvider_KVM, + }; + fRc = fRc and oSession.setParavirtProvider(adParavirtProviders[sParavirtMode]); + + fCfg64Bit = self.is64bitRequired() or (self.is64bit() and fHostSupports64bit and sVirtMode != 'raw'); + fRc = fRc and oSession.enableLongMode(fCfg64Bit); + if fCfg64Bit: # This is to avoid GUI pedantic warnings in the GUI. Sigh. + oOsType = oSession.getOsType(); + if oOsType is not None: + if oOsType.is64Bit and sVirtMode == 'raw': + assert(oOsType.id[-3:] == '_64'); + fRc = fRc and oSession.setOsType(oOsType.id[:-3]); + elif not oOsType.is64Bit and sVirtMode != 'raw': + fRc = fRc and oSession.setOsType(oOsType.id + '_64'); + + # New serial raw file. + if fRc and self.fCom1RawFile: + self.sCom1RawFile = self._generateRawPortFilename(oTestDrv, '-com1-', '.out'); + utils.noxcptDeleteFile(self.sCom1RawFile); + fRc = oSession.setupSerialToRawFile(0, self.sCom1RawFile); + + # Make life simpler for child classes. + if fRc: + fRc = self._childVmReconfig(oTestDrv, oVM, oSession); + + fRc = fRc and oSession.saveSettings(); + if not oSession.close(): + fRc = False; + if fRc is True: + return (True, oVM); + return (fRc, None); + + def _childVmReconfig(self, oTestDrv, oVM, oSession): + """ Hook into getReconfiguredVm() for children. """ + _ = oTestDrv; _ = oVM; _ = oSession; + return True; + + def getGuestArch(self): + """ Same as util.getHostArch. """ + return 'amd64' if self.sKind.find('_64') >= 0 else 'x86'; + + def getGuestOs(self): + """ Same as util.getHostOs. """ + if self.isWindows(): return 'win'; + if self.isOS2(): return 'os2'; + if self.isLinux(): return 'linux'; + reporter.error('getGuestOs does not what to return!'); + raise Exception(); + + def getGuestExeSuff(self): + """ The executable image suffix for the guest. """ + if self.isWindows() or self.isOS2(): + return '.exe'; + return ''; + + def getGuestOsDotArch(self): + """ Same as util.getHostOsDotArch.""" + return self.getGuestOs() + '.' + self.getGuestArch(); + + def isWindows(self): + """ Checks if it's a Windows VM. """ + return self.sGuestOsType == g_ksGuestOsTypeWindows; + + def isOS2(self): + """ Checks if it's an OS/2 VM. """ + return self.sGuestOsType == g_ksGuestOsTypeOS2; + + def isLinux(self): + """ Checks if it's an Linux VM. """ + return self.sGuestOsType == g_ksGuestOsTypeLinux; + + def is64bit(self): + """ Checks if it's a 64-bit VM. """ + return self.sKind.find('_64') >= 0; + + def is64bitRequired(self): + """ Check if 64-bit is required or not. """ + return (self.aInfo[g_iFlags] & g_k64) != 0; + + def isLoggedOntoDesktop(self): + """ Checks if the test VM is logging onto a graphical desktop by default. """ + if self.isWindows(): + return True; + if self.isOS2(): + return True; + if self.sVmName.find('-desktop'): + return True; + return False; + + def isViaIncompatible(self): + """ + Identifies VMs that doesn't work on VIA. + + Returns True if NOT supported on VIA, False if it IS supported. + """ + # Oracle linux doesn't like VIA in our experience + if self.aInfo[g_iKind] in ['Oracle', 'Oracle_64']: + return True; + # OS/2: "The system detected an internal processing error at location + # 0168:fff1da1f - 000e:ca1f. 0a8606fd + if self.isOS2(): + return True; + # Windows NT4 before SP4 won't work because of cmpxchg8b not being + # detected, leading to a STOP 3e(80,0,0,0). + if self.aInfo[g_iKind] == 'WindowsNT4': + if self.sVmName.find('sp') < 0: + return True; # no service pack. + if self.sVmName.find('sp0') >= 0 \ + or self.sVmName.find('sp1') >= 0 \ + or self.sVmName.find('sp2') >= 0 \ + or self.sVmName.find('sp3') >= 0: + return True; + # XP x64 on a physical VIA box hangs exactly like a VM. + if self.aInfo[g_iKind] in ['WindowsXP_64', 'Windows2003_64']: + return True; + # Vista 64 throws BSOD 0x5D (UNSUPPORTED_PROCESSOR) + if self.aInfo[g_iKind] in ['WindowsVista_64']: + return True; + # Solaris 11 hangs on VIA, tested on a physical box (testboxvqc) + if self.aInfo[g_iKind] in ['Solaris11_64']: + return True; + return False; + + def isShanghaiIncompatible(self): + """ + Identifies VMs that doesn't work on Shanghai. + + Returns True if NOT supported on Shanghai, False if it IS supported. + """ + # For now treat it just like VIA, to be adjusted later + return self.isViaIncompatible() + + def isP4Incompatible(self): + """ + Identifies VMs that doesn't work on Pentium 4 / Pentium D. + + Returns True if NOT supported on P4, False if it IS supported. + """ + # Stupid 1 kHz timer. Too much for antique CPUs. + if self.sVmName.find('rhel5') >= 0: + return True; + # Due to the boot animation the VM takes forever to boot. + if self.aInfo[g_iKind] == 'Windows2000': + return True; + return False; + + def isHostCpuAffectedByUbuntuNewAmdBug(self, oTestDrv): + """ + Checks if the host OS is affected by older ubuntu installers being very + picky about which families of AMD CPUs it would run on. + + The installer checks for family 15, later 16, later 20, and in 11.10 + they remove the family check for AMD CPUs. + """ + if not oTestDrv.isHostCpuAmd(): + return False; + try: + (uMaxExt, _, _, _) = oTestDrv.oVBox.host.getProcessorCPUIDLeaf(0, 0x80000000, 0); + (uFamilyModel, _, _, _) = oTestDrv.oVBox.host.getProcessorCPUIDLeaf(0, 0x80000001, 0); + except: + reporter.logXcpt(); + return False; + if uMaxExt < 0x80000001 or uMaxExt > 0x8000ffff: + return False; + + uFamily = (uFamilyModel >> 8) & 0xf + if uFamily == 0xf: + uFamily = ((uFamilyModel >> 20) & 0x7f) + 0xf; + ## @todo Break this down into which old ubuntu release supports exactly + ## which AMD family, if we care. + if uFamily <= 15: + return False; + reporter.log('Skipping "%s" because host CPU is a family %u AMD, which may cause trouble for the guest OS installer.' + % (self.sVmName, uFamily,)); + return True; + + def getTestUser(self): + """ + Gets the primary test user name. + """ + if self.isWindows(): + return 'Administrator'; + return 'vbox'; + + def getTestUserPassword(self, sUser = None): + """ + Gets the password for the primary user (or other specified one). + """ + if sUser == 'test': + return ''; + if sUser == 'vboxuser': # Default unattended installation user and password. + return 'changeme'; + return 'password'; + + def pathJoin(self, sBase, *asAppend): + """ See common.pathutils.joinEx(). """ + return pathutils.joinEx(self.isWindows() or self.isOS2(), sBase, *asAppend); + + def pathSep(self): + """ Returns the preferred paths separator for the guest OS. """ + return '\\' if self.isWindows() or self.isOS2() else '/'; + + +class BootSectorTestVm(TestVm): + """ + A Boot Sector Test VM. + """ + + def __init__(self, oSet, sVmName, sFloppy = None, asVirtModesSup = None, f64BitRequired = False): + self.f64BitRequired = f64BitRequired; + if asVirtModesSup is None: + asVirtModesSup = list(g_asVirtModes); + TestVm.__init__(self, sVmName, + oSet = oSet, + acCpusSup = [1,], + sFloppy = sFloppy, + asVirtModesSup = asVirtModesSup, + fPae = True, + fIoApic = True, + fVmmDevTestingPart = True, + fVmmDevTestingMmio = True, + ); + + def is64bitRequired(self): + return self.f64BitRequired; + + +class AncientTestVm(TestVm): + """ + A ancient Test VM, using the serial port for communicating results. + + We're looking for 'PASSED' and 'FAILED' lines in the COM1 output. + """ + + + def __init__(self, # pylint: disable=too-many-arguments + sVmName, # type: str + fGrouping = g_kfGrpAncient | g_kfGrpNoTxs, # type: int + sHd = None, # type: str + sKind = None, # type: str + acCpusSup = None, # type: List[int] + asVirtModesSup = None, # type: List[str] + sNic0AttachType = None, # type: str + sFloppy = None, # type: str + sFirmwareType = 'bios', # type: str + sChipsetType = 'piix3', # type: str + sHddControllerName = 'IDE Controller', # type: str + sDvdControllerName = 'IDE Controller', # type: str + cMBRamMax = None, # type: int + ): + TestVm.__init__(self, + sVmName, + fGrouping = fGrouping, + sHd = sHd, + sKind = sKind, + acCpusSup = [1] if acCpusSup is None else acCpusSup, + asVirtModesSup = asVirtModesSup, + sNic0AttachType = sNic0AttachType, + sFloppy = sFloppy, + sFirmwareType = sFirmwareType, + sChipsetType = sChipsetType, + sHddControllerType = sHddControllerName, + sDvdControllerType = sDvdControllerName, + asParavirtModesSup = (g_ksParavirtProviderNone,) + ); + self.fCom1RawFile = True; + self.cMBRamMax= cMBRamMax; + + + def _childVmReconfig(self, oTestDrv, oVM, oSession): + _ = oVM; _ = oTestDrv; + fRc = True; + + # DOS 4.01 doesn't like the default 32MB of memory. + if fRc and self.cMBRamMax is not None: + try: + cMBRam = oSession.o.machine.memorySize; + except: + cMBRam = self.cMBRamMax + 4; + if self.cMBRamMax < cMBRam: + fRc = oSession.setRamSize(self.cMBRamMax); + + return fRc; + + +class TestVmSet(object): + """ + A set of Test VMs. + """ + + def __init__(self, oTestVmManager = None, acCpus = None, asVirtModes = None, fIgnoreSkippedVm = False): + self.oTestVmManager = oTestVmManager; + if acCpus is None: + acCpus = [1, 2]; + self.acCpusDef = acCpus; + self.acCpus = acCpus; + if asVirtModes is None: + asVirtModes = list(g_asVirtModes); + self.asVirtModesDef = asVirtModes; + self.asVirtModes = asVirtModes; + self.aoTestVms = [] # type: list(BaseTestVm) + self.fIgnoreSkippedVm = fIgnoreSkippedVm; + self.asParavirtModes = None; ##< If None, use the first PV mode of the test VM, otherwise all modes in this list. + + def findTestVmByName(self, sVmName): + """ + Returns the TestVm object with the given name. + Returns None if not found. + """ + + # The 'tst-' prefix is optional. + sAltName = sVmName if sVmName.startswith('tst-') else 'tst-' + sVmName; + + for oTestVm in self.aoTestVms: + if oTestVm.sVmName in (sVmName, sAltName): + return oTestVm; + return None; + + def getAllVmNames(self, sSep = ':'): + """ + Returns names of all the test VMs in the set separated by + sSep (defaults to ':'). + """ + sVmNames = ''; + for oTestVm in self.aoTestVms: + sName = oTestVm.sVmName; + if sName.startswith('tst-'): + sName = sName[4:]; + if sVmNames == '': + sVmNames = sName; + else: + sVmNames = sVmNames + sSep + sName; + return sVmNames; + + def showUsage(self): + """ + Invoked by vbox.TestDriver. + """ + reporter.log(''); + reporter.log('Test VM selection and general config options:'); + reporter.log(' --virt-modes <m1[:m2[:...]]>'); + reporter.log(' Default: %s' % (':'.join(self.asVirtModesDef))); + reporter.log(' --skip-virt-modes <m1[:m2[:...]]>'); + reporter.log(' Use this to avoid hwvirt or hwvirt-np when not supported by the host'); + reporter.log(' since we cannot detect it using the main API. Use after --virt-modes.'); + reporter.log(' --cpu-counts <c1[:c2[:...]]>'); + reporter.log(' Default: %s' % (':'.join(str(c) for c in self.acCpusDef))); + reporter.log(' --test-vms <vm1[:vm2[:...]]>'); + reporter.log(' Test the specified VMs in the given order. Use this to change'); + reporter.log(' the execution order or limit the choice of VMs'); + reporter.log(' Default: %s (all)' % (self.getAllVmNames(),)); + reporter.log(' --skip-vms <vm1[:vm2[:...]]>'); + reporter.log(' Skip the specified VMs when testing.'); + reporter.log(' --snapshot-restore-current'); + reporter.log(' Restores the current snapshot and resumes execution.'); + reporter.log(' --paravirt-modes <pv1[:pv2[:...]]>'); + reporter.log(' Set of paravirtualized providers (modes) to tests. Intersected with what the test VM supports.'); + reporter.log(' Default is the first PV mode the test VMs support, generally same as "legacy".'); + reporter.log(' --with-nested-hwvirt-only'); + reporter.log(' Test VMs using nested hardware-virtualization only.'); + reporter.log(' --without-nested-hwvirt-only'); + reporter.log(' Test VMs not using nested hardware-virtualization only.'); + ## @todo Add more options for controlling individual VMs. + return True; + + def parseOption(self, asArgs, iArg): + """ + Parses the set test vm set options (--test-vms and --skip-vms), modifying the set + Invoked by the testdriver method with the same name. + + Keyword arguments: + asArgs -- The argument vector. + iArg -- The index of the current argument. + + Returns iArg if the option was not recognized and the caller should handle it. + Returns the index of the next argument when something is consumed. + + In the event of a syntax error, a InvalidOption or QuietInvalidOption + is thrown. + """ + + if asArgs[iArg] == '--virt-modes': + iArg += 1; + if iArg >= len(asArgs): + raise base.InvalidOption('The "--virt-modes" takes a colon separated list of modes'); + + self.asVirtModes = asArgs[iArg].split(':'); + for s in self.asVirtModes: + if s not in self.asVirtModesDef: + raise base.InvalidOption('The "--virt-modes" value "%s" is not valid; valid values are: %s' \ + % (s, ' '.join(self.asVirtModesDef))); + + elif asArgs[iArg] == '--skip-virt-modes': + iArg += 1; + if iArg >= len(asArgs): + raise base.InvalidOption('The "--skip-virt-modes" takes a colon separated list of modes'); + + for s in asArgs[iArg].split(':'): + if s not in self.asVirtModesDef: + raise base.InvalidOption('The "--virt-modes" value "%s" is not valid; valid values are: %s' \ + % (s, ' '.join(self.asVirtModesDef))); + if s in self.asVirtModes: + self.asVirtModes.remove(s); + + elif asArgs[iArg] == '--cpu-counts': + iArg += 1; + if iArg >= len(asArgs): + raise base.InvalidOption('The "--cpu-counts" takes a colon separated list of cpu counts'); + + self.acCpus = []; + for s in asArgs[iArg].split(':'): + try: c = int(s); + except: raise base.InvalidOption('The "--cpu-counts" value "%s" is not an integer' % (s,)); + if c <= 0: raise base.InvalidOption('The "--cpu-counts" value "%s" is zero or negative' % (s,)); + self.acCpus.append(c); + + elif asArgs[iArg] == '--test-vms': + iArg += 1; + if iArg >= len(asArgs): + raise base.InvalidOption('The "--test-vms" takes colon separated list'); + + for oTestVm in self.aoTestVms: + oTestVm.fSkip = True; + + asTestVMs = asArgs[iArg].split(':'); + for s in asTestVMs: + oTestVm = self.findTestVmByName(s); + if oTestVm is None: + raise base.InvalidOption('The "--test-vms" value "%s" is not valid; valid values are: %s' \ + % (s, self.getAllVmNames(' '))); + oTestVm.fSkip = False; + + elif asArgs[iArg] == '--skip-vms': + iArg += 1; + if iArg >= len(asArgs): + raise base.InvalidOption('The "--skip-vms" takes colon separated list'); + + asTestVMs = asArgs[iArg].split(':'); + for s in asTestVMs: + oTestVm = self.findTestVmByName(s); + if oTestVm is None: + reporter.log('warning: The "--test-vms" value "%s" does not specify any of our test VMs.' % (s,)); + else: + oTestVm.fSkip = True; + + elif asArgs[iArg] == '--snapshot-restore-current': + for oTestVm in self.aoTestVms: + if oTestVm.fSkip is False: + oTestVm.fSnapshotRestoreCurrent = True; + reporter.log('VM "%s" will be restored.' % (oTestVm.sVmName)); + + elif asArgs[iArg] == '--paravirt-modes': + iArg += 1 + if iArg >= len(asArgs): + raise base.InvalidOption('The "--paravirt-modes" takes a colon separated list of modes'); + + self.asParavirtModes = asArgs[iArg].split(':') + for sPvMode in self.asParavirtModes: + if sPvMode not in g_kasParavirtProviders: + raise base.InvalidOption('The "--paravirt-modes" value "%s" is not valid; valid values are: %s' + % (sPvMode, ', '.join(g_kasParavirtProviders),)); + if not self.asParavirtModes: + self.asParavirtModes = None; + + # HACK ALERT! Reset the random paravirt selection for members. + for oTestVm in self.aoTestVms: + oTestVm.asParavirtModesSup = oTestVm.asParavirtModesSupOrg; + + elif asArgs[iArg] == '--with-nested-hwvirt-only': + for oTestVm in self.aoTestVms: + if oTestVm.fNstHwVirt is False: + oTestVm.fSkip = True; + + elif asArgs[iArg] == '--without-nested-hwvirt-only': + for oTestVm in self.aoTestVms: + if oTestVm.fNstHwVirt is True: + oTestVm.fSkip = True; + + else: + return iArg; + return iArg + 1; + + def getResourceSet(self): + """ + Called vbox.TestDriver.getResourceSet and returns a list of paths of resources. + """ + asResources = []; + for oTestVm in self.aoTestVms: + if not oTestVm.fSkip: + if isinstance(oTestVm, BaseTestVm): # Temporarily... + asResources.extend(oTestVm.getResourceSet()); + else: + if oTestVm.sHd is not None: + asResources.append(oTestVm.sHd); + if oTestVm.sDvdImage is not None: + asResources.append(oTestVm.sDvdImage); + return asResources; + + def actionConfig(self, oTestDrv, eNic0AttachType = None, sDvdImage = None): + """ + For base.TestDriver.actionConfig. Configure the VMs with defaults and + a few tweaks as per arguments. + + Returns True if successful. + Returns False if not. + """ + + for oTestVm in self.aoTestVms: + if oTestVm.fSkip: + continue; + if oTestVm.skipCreatingVm(oTestDrv): + oTestVm.fSkip = True; + continue; + + if oTestVm.fSnapshotRestoreCurrent: + # If we want to restore a VM we don't need to create + # the machine anymore -- so just add it to the test VM list. + oVM = oTestDrv.addTestMachine(oTestVm.sVmName); + else: + oVM = oTestVm.createVm(oTestDrv, eNic0AttachType, sDvdImage); + if oVM is None: + return False; + + return True; + + def _removeUnsupportedVirtModes(self, oTestDrv): + """ + Removes unsupported virtualization modes. + """ + if 'hwvirt' in self.asVirtModes and not oTestDrv.hasHostHwVirt(): + reporter.log('Hardware assisted virtualization is not available on the host, skipping it.'); + self.asVirtModes.remove('hwvirt'); + + if 'hwvirt-np' in self.asVirtModes and not oTestDrv.hasHostNestedPaging(): + reporter.log('Nested paging not supported by the host, skipping it.'); + self.asVirtModes.remove('hwvirt-np'); + + if 'raw' in self.asVirtModes and not oTestDrv.hasRawModeSupport(): + reporter.log('Raw-mode virtualization is not available in this build (or perhaps for this host), skipping it.'); + self.asVirtModes.remove('raw'); + + return True; + + def actionExecute(self, oTestDrv, fnCallback): # pylint: disable=too-many-locals + """ + For base.TestDriver.actionExecute. Calls the callback function for + each of the VMs and basic configuration variations (virt-mode and cpu + count). + + Returns True if all fnCallback calls returned True, otherwise False. + + The callback can return True, False or None. The latter is for when the + test is skipped. (True is for success, False is for failure.) + """ + + self._removeUnsupportedVirtModes(oTestDrv); + cMaxCpus = oTestDrv.getHostCpuCount(); + + # + # The test loop. + # + fRc = True; + for oTestVm in self.aoTestVms: + if oTestVm.fSkip and self.fIgnoreSkippedVm: + reporter.log2('Ignoring VM %s (fSkip = True).' % (oTestVm.sVmName,)); + continue; + reporter.testStart(oTestVm.sVmName); + if oTestVm.fSkip: + reporter.testDone(fSkipped = True); + continue; + + # Intersect the supported modes and the ones being testing. + asVirtModesSup = [sMode for sMode in oTestVm.asVirtModesSup if sMode in self.asVirtModes]; + + # Ditto for CPUs. + acCpusSup = [cCpus for cCpus in oTestVm.acCpusSup if cCpus in self.acCpus]; + + # Ditto for paravirtualization modes, except if not specified we got a less obvious default. + if self.asParavirtModes is not None and oTestDrv.fpApiVer >= 5.0: + asParavirtModes = [sPvMode for sPvMode in oTestVm.asParavirtModesSup if sPvMode in self.asParavirtModes]; + assert None not in asParavirtModes; + elif oTestDrv.fpApiVer >= 5.0: + asParavirtModes = (oTestVm.asParavirtModesSup[0],); + assert asParavirtModes[0] is not None; + else: + asParavirtModes = (None,); + + for cCpus in acCpusSup: + if cCpus == 1: + reporter.testStart('1 cpu'); + else: + reporter.testStart('%u cpus' % (cCpus)); + if cCpus > cMaxCpus: + reporter.testDone(fSkipped = True); + continue; + + cTests = 0; + for sVirtMode in asVirtModesSup: + if sVirtMode == 'raw' and cCpus > 1: + continue; + reporter.testStart('%s' % ( g_dsVirtModeDescs[sVirtMode], ) ); + cStartTests = cTests; + + for sParavirtMode in asParavirtModes: + if sParavirtMode is not None: + assert oTestDrv.fpApiVer >= 5.0; + reporter.testStart('%s' % ( sParavirtMode, ) ); + + # Reconfigure the VM. + try: + (rc2, oVM) = oTestVm.getReconfiguredVm(oTestDrv, cCpus, sVirtMode, sParavirtMode = sParavirtMode); + except KeyboardInterrupt: + raise; + except: + reporter.errorXcpt(cFrames = 9); + rc2 = False; + if rc2 is True: + # Do the testing. + try: + rc2 = fnCallback(oVM, oTestVm); + except KeyboardInterrupt: + raise; + except: + reporter.errorXcpt(cFrames = 9); + rc2 = False; + if rc2 is False: + reporter.maybeErr(reporter.testErrorCount() == 0, 'fnCallback failed'); + elif rc2 is False: + reporter.log('getReconfiguredVm failed'); + if rc2 is False: + fRc = False; + + cTests = cTests + (rc2 is not None); + if sParavirtMode is not None: + reporter.testDone(fSkipped = (rc2 is None)); + + reporter.testDone(fSkipped = cTests == cStartTests); + + reporter.testDone(fSkipped = cTests == 0); + + _, cErrors = reporter.testDone(); + if cErrors > 0: + fRc = False; + return fRc; + + def enumerateTestVms(self, fnCallback): + """ + Enumerates all the 'active' VMs. + + Returns True if all fnCallback calls returned True. + Returns False if any returned False. + Returns None immediately if fnCallback returned None. + """ + fRc = True; + for oTestVm in self.aoTestVms: + if not oTestVm.fSkip: + fRc2 = fnCallback(oTestVm); + if fRc2 is None: + return fRc2; + fRc = fRc and fRc2; + return fRc; + + + +class TestVmManager(object): + """ + Test VM manager. + """ + + ## @name VM grouping flags + ## @{ + kfGrpSmoke = g_kfGrpSmoke; + kfGrpStandard = g_kfGrpStandard; + kfGrpStdSmoke = g_kfGrpStdSmoke; + kfGrpWithGAs = g_kfGrpWithGAs; + kfGrpNoTxs = g_kfGrpNoTxs; + kfGrpAncient = g_kfGrpAncient; + kfGrpExotic = g_kfGrpExotic; + ## @} + + kaTestVMs = ( + # Note: The images in the 6.1 folder all have been pre-configured to allow for Guest Additions installation + # (come with build essentials, kernel headers). + # Linux + TestVm('tst-ubuntu-18_04_3-64', kfGrpStdSmoke, sHd = '6.1/ubuntu-18_04_3-amd64-2.vdi', + sKind = 'Ubuntu_64', acCpusSup = range(1, 33), fIoApic = True, + asParavirtModesSup = [g_ksParavirtProviderKVM,]), + # Note: Deprecated; had SELinux + Screensaver (black screen) enabled. + #TestVm('tst-ol-8_1-64-efi', kfGrpStdSmoke, sHd = '6.1/efi/ol-8_1-efi-amd64.vdi', + # sKind = 'Oracle_64', acCpusSup = range(1, 33), fIoApic = True, sFirmwareType = 'efi', + # asParavirtModesSup = [g_ksParavirtProviderKVM,]), + TestVm('tst-ol-8_1-64-efi', kfGrpStdSmoke, sHd = '6.1/efi/ol-8_1-efi-amd64-2.vdi', + sKind = 'Oracle_64', acCpusSup = range(1, 33), fIoApic = True, sFirmwareType = 'efi', + asParavirtModesSup = [g_ksParavirtProviderKVM,]), + TestVm('tst-ol-6u2-32', kfGrpStdSmoke, sHd = '6.1/ol-6u2-x86.vdi', + sKind = 'Oracle', acCpusSup = range(1, 33), fIoApic = True, + asParavirtModesSup = [g_ksParavirtProviderKVM,]), + TestVm('tst-ubuntu-15_10-64-efi', kfGrpStdSmoke, sHd = '6.1/efi/ubuntu-15_10-efi-amd64-3.vdi', + sKind = 'Ubuntu_64', acCpusSup = range(1, 33), fIoApic = True, sFirmwareType = 'efi', + asParavirtModesSup = [g_ksParavirtProviderKVM,]), + # Note: Deprecated / buggy; use the one in the 6.1 folder. + #TestVm('tst-ubuntu-15_10-64-efi', kfGrpStdSmoke, sHd = '4.2/efi/ubuntu-15_10-efi-amd64.vdi', + # sKind = 'Ubuntu_64', acCpusSup = range(1, 33), fIoApic = True, sFirmwareType = 'efi', + # asParavirtModesSup = [g_ksParavirtProviderKVM,]), + TestVm('tst-rhel5', kfGrpSmoke, sHd = '3.0/tcp/rhel5.vdi', + sKind = 'RedHat', acCpusSup = range(1, 33), fIoApic = True, sNic0AttachType = 'nat'), + TestVm('tst-arch', kfGrpStandard, sHd = '4.2/usb/tst-arch.vdi', + sKind = 'ArchLinux_64', acCpusSup = range(1, 33), fIoApic = True, sNic0AttachType = 'nat'), + # disabled 2019-03-08 klaus - fails all over the place and pollutes the test results + #TestVm('tst-ubuntu-1804-64', kfGrpStdSmoke, sHd = '4.2/ubuntu-1804/t-ubuntu-1804-64.vdi', + # sKind = 'Ubuntu_64', acCpusSup = range(1, 33), fIoApic = True), + TestVm('tst-ol76-64', kfGrpStdSmoke, sHd = '4.2/ol76/t-ol76-64.vdi', + sKind = 'Oracle_64', acCpusSup = range(1, 33), fIoApic = True), + TestVm('tst-ubuntu-20_04-64-amdvi', kfGrpStdSmoke, sHd = '6.1/ubuntu-20_04-64.vdi', + sKind = 'Ubuntu_64', acCpusSup = range(1, 33), fIoApic = True, + asParavirtModesSup = [g_ksParavirtProviderKVM,], sNic0AttachType = 'nat', sChipsetType = 'ich9', + sIommuType = 'amd'), + TestVm('tst-ubuntu-20_04-64-vtd', kfGrpStdSmoke, sHd = '6.1/ubuntu-20_04-64.vdi', + sKind = 'Ubuntu_64', acCpusSup = range(1, 33), fIoApic = True, + asParavirtModesSup = [g_ksParavirtProviderKVM,], sNic0AttachType = 'nat', sChipsetType = 'ich9', + sIommuType = 'intel'), + + # Solaris + TestVm('tst-sol10', kfGrpSmoke, sHd = '3.0/tcp/solaris10.vdi', + sKind = 'Solaris', acCpusSup = range(1, 33), fPae = True, sNic0AttachType = 'bridged'), + TestVm('tst-sol10-64', kfGrpSmoke, sHd = '3.0/tcp/solaris10.vdi', + sKind = 'Solaris_64', acCpusSup = range(1, 33), sNic0AttachType = 'bridged'), + TestVm('tst-sol11u1', kfGrpSmoke, sHd = '4.2/nat/sol11u1/t-sol11u1.vdi', + sKind = 'Solaris11_64', acCpusSup = range(1, 33), sNic0AttachType = 'nat', fIoApic = True, + sHddControllerType = 'SATA Controller'), + #TestVm('tst-sol11u1-ich9', kfGrpSmoke, sHd = '4.2/nat/sol11u1/t-sol11u1.vdi', + # sKind = 'Solaris11_64', acCpusSup = range(1, 33), sNic0AttachType = 'nat', fIoApic = True, + # sHddControllerType = 'SATA Controller', sChipsetType = 'ich9'), + + # NT 3.x + TestVm('tst-nt310', kfGrpAncient, sHd = '5.2/great-old-ones/t-nt310/t-nt310.vdi', + sKind = 'WindowsNT3x', acCpusSup = [1], sHddControllerType = 'BusLogic SCSI Controller', + sDvdControllerType = 'BusLogic SCSI Controller'), + TestVm('tst-nt350', kfGrpAncient, sHd = '5.2/great-old-ones/t-nt350/t-nt350.vdi', + sKind = 'WindowsNT3x', acCpusSup = [1], sHddControllerType = 'BusLogic SCSI Controller', + sDvdControllerType = 'BusLogic SCSI Controller'), + TestVm('tst-nt351', kfGrpAncient, sHd = '5.2/great-old-ones/t-nt350/t-nt351.vdi', + sKind = 'WindowsNT3x', acCpusSup = [1], sHddControllerType = 'BusLogic SCSI Controller', + sDvdControllerType = 'BusLogic SCSI Controller'), + + # NT 4 + TestVm('tst-nt4sp1', kfGrpStdSmoke, sHd = '4.2/nat/nt4sp1/t-nt4sp1.vdi', + sKind = 'WindowsNT4', acCpusSup = [1], sNic0AttachType = 'nat'), + + TestVm('tst-nt4sp6', kfGrpStdSmoke, sHd = '4.2/nt4sp6/t-nt4sp6.vdi', + sKind = 'WindowsNT4', acCpusSup = range(1, 33)), + + # W2K + TestVm('tst-w2ksp4', kfGrpStdSmoke, sHd = '4.2/win2ksp4/t-win2ksp4.vdi', + sKind = 'Windows2000', acCpusSup = range(1, 33)), + + # XP + TestVm('tst-xppro', kfGrpStdSmoke, sHd = '4.2/nat/xppro/t-xppro.vdi', + sKind = 'WindowsXP', acCpusSup = range(1, 33), sNic0AttachType = 'nat'), + TestVm('tst-xpsp2', kfGrpStdSmoke, sHd = '4.2/xpsp2/t-winxpsp2.vdi', + sKind = 'WindowsXP', acCpusSup = range(1, 33), fIoApic = True), + TestVm('tst-xpsp2-halaacpi', kfGrpStdSmoke, sHd = '4.2/xpsp2/t-winxp-halaacpi.vdi', + sKind = 'WindowsXP', acCpusSup = range(1, 33), fIoApic = True), + TestVm('tst-xpsp2-halacpi', kfGrpStdSmoke, sHd = '4.2/xpsp2/t-winxp-halacpi.vdi', + sKind = 'WindowsXP', acCpusSup = range(1, 33), fIoApic = True), + TestVm('tst-xpsp2-halapic', kfGrpStdSmoke, sHd = '4.2/xpsp2/t-winxp-halapic.vdi', + sKind = 'WindowsXP', acCpusSup = range(1, 33), fIoApic = True), + TestVm('tst-xpsp2-halmacpi', kfGrpStdSmoke, sHd = '4.2/xpsp2/t-winxp-halmacpi.vdi', + sKind = 'WindowsXP', acCpusSup = range(2, 33), fIoApic = True), + TestVm('tst-xpsp2-halmps', kfGrpStdSmoke, sHd = '4.2/xpsp2/t-winxp-halmps.vdi', + sKind = 'WindowsXP', acCpusSup = range(2, 33), fIoApic = True), + + # W2K3 + TestVm('tst-win2k3ent', kfGrpSmoke, sHd = '3.0/tcp/win2k3ent-acpi.vdi', + sKind = 'Windows2003', acCpusSup = range(1, 33), fPae = True, sNic0AttachType = 'bridged'), + + # W7 + TestVm('tst-win7', kfGrpStdSmoke, sHd = '6.1/win7-32/t-win7-32-1.vdi', + sKind = 'Windows7', acCpusSup = range(1, 33), fIoApic = True), + # Note: Deprecated due to activation issues; use t-win7-32-1 instead. + #TestVm('tst-win7', kfGrpStdSmoke, sHd = '6.1/win7-32/t-win7-32.vdi', + # sKind = 'Windows7', acCpusSup = range(1, 33), fIoApic = True), + # Note: Deprecated; use the one in the 6.1 folder. + #TestVm('tst-win7', kfGrpStdSmoke, sHd = '4.2/win7-32/t-win7.vdi', + # sKind = 'Windows7', acCpusSup = range(1, 33), fIoApic = True), + + # W8 + TestVm('tst-win8-64', kfGrpStdSmoke, sHd = '4.2/win8-64/t-win8-64.vdi', + sKind = 'Windows8_64', acCpusSup = range(1, 33), fIoApic = True), + #TestVm('tst-win8-64-ich9', kfGrpStdSmoke, sHd = '4.2/win8-64/t-win8-64.vdi', + # sKind = 'Windows8_64', acCpusSup = range(1, 33), fIoApic = True, sChipsetType = 'ich9'), + + # W10 + TestVm('tst-win10-efi', kfGrpStdSmoke, sHd = '4.2/efi/win10-efi-x86.vdi', + sKind = 'Windows10', acCpusSup = range(1, 33), fIoApic = True, sFirmwareType = 'efi'), + TestVm('tst-win10-64-efi', kfGrpStdSmoke, sHd = '4.2/efi/win10-efi-amd64.vdi', + sKind = 'Windows10_64', acCpusSup = range(1, 33), fIoApic = True, sFirmwareType = 'efi'), + #TestVm('tst-win10-64-efi-ich9', kfGrpStdSmoke, sHd = '4.2/efi/win10-efi-amd64.vdi', + # sKind = 'Windows10_64', acCpusSup = range(1, 33), fIoApic = True, sFirmwareType = 'efi', sChipsetType = 'ich9'), + + # Nested hardware-virtualization + TestVm('tst-nsthwvirt-ubuntu-64', kfGrpStdSmoke, sHd = '5.3/nat/nsthwvirt-ubuntu64/t-nsthwvirt-ubuntu64.vdi', + sKind = 'Ubuntu_64', acCpusSup = range(1, 2), asVirtModesSup = ['hwvirt-np',], fIoApic = True, fNstHwVirt = True, + sNic0AttachType = 'nat'), + + # Audio testing. + TestVm('tst-audio-debian10-64', kfGrpStdSmoke, sHd = '6.1/audio/debian10-amd64-7.vdi', + sKind = 'Debian_64', acCpusSup = range(1, 33), fIoApic = True), + + # DOS and Old Windows. + AncientTestVm('tst-dos20', sKind = 'DOS', + sHd = '5.2/great-old-ones/t-dos20/t-dos20.vdi'), + AncientTestVm('tst-dos401-win30me', sKind = 'DOS', + sHd = '5.2/great-old-ones/t-dos401-win30me/t-dos401-win30me.vdi', cMBRamMax = 4), + AncientTestVm('tst-dos401-emm386-win30me', sKind = 'DOS', + sHd = '5.2/great-old-ones/t-dos401-emm386-win30me/t-dos401-emm386-win30me.vdi', cMBRamMax = 4), + AncientTestVm('tst-dos50-win31', sKind = 'DOS', + sHd = '5.2/great-old-ones/t-dos50-win31/t-dos50-win31.vdi'), + AncientTestVm('tst-dos50-emm386-win31', sKind = 'DOS', + sHd = '5.2/great-old-ones/t-dos50-emm386-win31/t-dos50-emm386-win31.vdi'), + AncientTestVm('tst-dos622', sKind = 'DOS', + sHd = '5.2/great-old-ones/t-dos622/t-dos622.vdi'), + AncientTestVm('tst-dos622-emm386', sKind = 'DOS', + sHd = '5.2/great-old-ones/t-dos622-emm386/t-dos622-emm386.vdi'), + AncientTestVm('tst-dos71', sKind = 'DOS', + sHd = '5.2/great-old-ones/t-dos71/t-dos71.vdi'), + + #AncientTestVm('tst-dos5-win311a', sKind = 'DOS', sHd = '5.2/great-old-ones/t-dos5-win311a/t-dos5-win311a.vdi'), + ); + + + def __init__(self, sResourcePath): + self.sResourcePath = sResourcePath; + + def selectSet(self, fGrouping, sTxsTransport = None, fCheckResources = True): + """ + Returns a VM set with the selected VMs. + """ + oSet = TestVmSet(oTestVmManager = self); + for oVm in self.kaTestVMs: + if oVm.fGrouping & fGrouping: + if sTxsTransport is None or oVm.sNic0AttachType is None or sTxsTransport == oVm.sNic0AttachType: + if not fCheckResources or not oVm.getMissingResources(self.sResourcePath): + oCopyVm = copy.deepcopy(oVm); + oCopyVm.oSet = oSet; + oSet.aoTestVms.append(oCopyVm); + return oSet; + + def getStandardVmSet(self, sTxsTransport): + """ + Gets the set of standard test VMs. + + This is supposed to do something seriously clever, like searching the + testrsrc tree for usable VMs, but for the moment it's all hard coded. :-) + """ + return self.selectSet(self.kfGrpStandard, sTxsTransport) + + def getSmokeVmSet(self, sTxsTransport = None): + """Gets a representative set of VMs for smoke testing. """ + return self.selectSet(self.kfGrpSmoke, sTxsTransport); + + def shutUpPyLint(self): + """ Shut up already! """ + return self.sResourcePath; diff --git a/src/VBox/ValidationKit/testdriver/vboxwrappers.py b/src/VBox/ValidationKit/testdriver/vboxwrappers.py new file mode 100755 index 00000000..075a19d2 --- /dev/null +++ b/src/VBox/ValidationKit/testdriver/vboxwrappers.py @@ -0,0 +1,3666 @@ +# -*- coding: utf-8 -*- +# $Id: vboxwrappers.py $ +# pylint: disable=too-many-lines + +""" +VirtualBox Wrapper Classes +""" + +__copyright__ = \ +""" +Copyright (C) 2010-2022 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 154728 $" + + +# Standard Python imports. +import os; +import socket; +import sys; + +# Validation Kit imports. +from common import utils; +from common import netutils; +from testdriver import base; +from testdriver import reporter; +from testdriver import txsclient; +from testdriver import vboxcon; +from testdriver import vbox; +from testdriver.base import TdTaskBase; + + +def _ControllerNameToBusAndType(sController): + """ Translate a controller name to a storage bus. """ + if sController == "IDE Controller": + eBus = vboxcon.StorageBus_IDE; + eType = vboxcon.StorageControllerType_PIIX4; + elif sController == "SATA Controller": + eBus = vboxcon.StorageBus_SATA; + eType = vboxcon.StorageControllerType_IntelAhci; + elif sController == "Floppy Controller": + eType = vboxcon.StorageControllerType_I82078; + eBus = vboxcon.StorageBus_Floppy; + elif sController == "SAS Controller": + eBus = vboxcon.StorageBus_SAS; + eType = vboxcon.StorageControllerType_LsiLogicSas; + elif sController == "SCSI Controller": + eBus = vboxcon.StorageBus_SCSI; + eType = vboxcon.StorageControllerType_LsiLogic; + elif sController == "BusLogic SCSI Controller": + eBus = vboxcon.StorageBus_SCSI; + eType = vboxcon.StorageControllerType_BusLogic; + elif sController == "NVMe Controller": + eBus = vboxcon.StorageBus_PCIe; + eType = vboxcon.StorageControllerType_NVMe; + elif sController == "VirtIO SCSI Controller": + eBus = vboxcon.StorageBus_VirtioSCSI; + eType = vboxcon.StorageControllerType_VirtioSCSI; + else: + eBus = vboxcon.StorageBus_Null; + eType = vboxcon.StorageControllerType_Null; + return (eBus, eType); + + +def _nameMachineState(eState): + """ Gets the name (string) of a machine state.""" + if eState == vboxcon.MachineState_PoweredOff: return 'PoweredOff'; + if eState == vboxcon.MachineState_Saved: return 'Saved'; + if eState == vboxcon.MachineState_Teleported: return 'Teleported'; + if eState == vboxcon.MachineState_Aborted: return 'Aborted'; + if eState == vboxcon.MachineState_Running: return 'Running'; + if eState == vboxcon.MachineState_Paused: return 'Paused'; + if eState == vboxcon.MachineState_Stuck: return 'GuruMeditation'; + if eState == vboxcon.MachineState_Teleporting: return 'Teleporting'; + if eState == vboxcon.MachineState_LiveSnapshotting: return 'LiveSnapshotting'; + if eState == vboxcon.MachineState_Starting: return 'Starting'; + if eState == vboxcon.MachineState_Stopping: return 'Stopping'; + if eState == vboxcon.MachineState_Saving: return 'Saving'; + if eState == vboxcon.MachineState_Restoring: return 'Restoring'; + if eState == vboxcon.MachineState_TeleportingPausedVM: return 'TeleportingPausedVM'; + if eState == vboxcon.MachineState_TeleportingIn: return 'TeleportingIn'; + if eState == vboxcon.MachineState_DeletingSnapshotOnline: return 'DeletingSnapshotOnline'; + if eState == vboxcon.MachineState_DeletingSnapshotPaused: return 'DeletingSnapshotPaused'; + if eState == vboxcon.MachineState_RestoringSnapshot: return 'RestoringSnapshot'; + if eState == vboxcon.MachineState_DeletingSnapshot: return 'DeletingSnapshot'; + if eState == vboxcon.MachineState_SettingUp: return 'SettingUp'; + if hasattr(vboxcon, 'MachineState_FaultTolerantSyncing'): + if eState == vboxcon.MachineState_FaultTolerantSyncing: return 'FaultTolerantSyncing'; + if hasattr(vboxcon, 'MachineState_AbortedSaved'): # since r147033 / 7.0 + if eState == vboxcon.MachineState_AbortedSaved: return 'Aborted-Saved'; + return 'Unknown-%s' % (eState,); + + +class VirtualBoxWrapper(object): # pylint: disable=too-few-public-methods + """ + Wrapper around the IVirtualBox object that adds some (hopefully) useful + utility methods + + The real object can be accessed thru the o member. That said, members can + be accessed directly as well. + """ + + def __init__(self, oVBox, oVBoxMgr, fpApiVer, oTstDrv): + self.o = oVBox; + self.oVBoxMgr = oVBoxMgr; + self.fpApiVer = fpApiVer; + self.oTstDrv = oTstDrv; + + def __getattr__(self, sName): + # Try ourselves first. + try: + oAttr = self.__dict__[sName]; + except: + #try: + # oAttr = dir(self)[sName]; + #except AttributeError: + oAttr = getattr(self.o, sName); + return oAttr; + + # + # Utilities. + # + + def registerDerivedEventHandler(self, oSubClass, dArgs = None): + """ + Create an instance of the given VirtualBoxEventHandlerBase sub-class + and register it. + + The new instance is returned on success. None is returned on error. + """ + dArgsCopy = dArgs.copy() if dArgs is not None else {}; + dArgsCopy['oVBox'] = self; + return oSubClass.registerDerivedEventHandler(self.oVBoxMgr, self.fpApiVer, oSubClass, dArgsCopy, + self.o, 'IVirtualBox', 'IVirtualBoxCallback'); + + def deleteHdByLocation(self, sHdLocation): + """ + Deletes a disk image from the host, given it's location. + Returns True on success and False on failure. Error information is logged. + """ + try: + oIMedium = self.o.findHardDisk(sHdLocation); + except: + try: + if self.fpApiVer >= 4.1: + oIMedium = self.o.openMedium(sHdLocation, vboxcon.DeviceType_HardDisk, + vboxcon.AccessMode_ReadWrite, False); + elif self.fpApiVer >= 4.0: + oIMedium = self.o.openMedium(sHdLocation, vboxcon.DeviceType_HardDisk, + vboxcon.AccessMode_ReadWrite); + else: + oIMedium = self.o.openHardDisk(sHdLocation, vboxcon.AccessMode_ReadOnly, False, "", False, ""); + except: + return reporter.errorXcpt('failed to open hd "%s"' % (sHdLocation)); + return self.deleteHdByMedium(oIMedium) + + def deleteHdByMedium(self, oIMedium): + """ + Deletes a disk image from the host, given an IMedium reference. + Returns True on success and False on failure. Error information is logged. + """ + try: oProgressCom = oIMedium.deleteStorage(); + except: return reporter.errorXcpt('deleteStorage() for disk %s failed' % (oIMedium,)); + try: oProgress = ProgressWrapper(oProgressCom, self.oVBoxMgr, self.oTstDrv, 'delete disk %s' % (oIMedium.location)); + except: return reporter.errorXcpt(); + oProgress.wait(); + oProgress.logResult(); + return oProgress.isSuccess(); + + + +class ProgressWrapper(TdTaskBase): + """ + Wrapper around a progress object for making it a task and providing useful + utility methods. + The real progress object can be accessed thru the o member. + """ + + def __init__(self, oProgress, oVBoxMgr, oTstDrv, sName): + TdTaskBase.__init__(self, utils.getCallerName()); + self.o = oProgress; + self.oVBoxMgr = oVBoxMgr; + self.oTstDrv = oTstDrv; + self.sName = sName; + + def toString(self): + return '<%s sName=%s, oProgress=%s >' \ + % (TdTaskBase.toString(self), self.sName, self.o); + + # + # TdTaskBase overrides. + # + + def pollTask(self, fLocked = False): + """ + Overrides TdTaskBase.pollTask(). + + This method returns False until the progress object has completed. + """ + self.doQuickApiTest(); + try: + try: + if self.o.completed: + return True; + except: + pass; + finally: + self.oTstDrv.processPendingEvents(); + return False; + + def waitForTask(self, cMsTimeout = 0): + """ + Overrides TdTaskBase.waitForTask(). + Process XPCOM/COM events while waiting. + """ + msStart = base.timestampMilli(); + fState = self.pollTask(False); + while not fState: + cMsElapsed = base.timestampMilli() - msStart; + if cMsElapsed > cMsTimeout: + break; + cMsToWait = cMsTimeout - cMsElapsed; + cMsToWait = min(cMsToWait, 500); + try: + self.o.waitForCompletion(cMsToWait); + except KeyboardInterrupt: raise; + except: pass; + if self.fnProcessEvents: + self.fnProcessEvents(); + reporter.doPollWork('ProgressWrapper.waitForTask'); + fState = self.pollTask(False); + return fState; + + # + # Utility methods. + # + + def isSuccess(self): + """ + Tests if the progress object completed successfully. + Returns True on success, False on failure or incomplete. + """ + if not self.isCompleted(): + return False; + return self.getResult() >= 0; + + def isCompleted(self): + """ + Wrapper around IProgress.completed. + """ + return self.pollTask(); + + def isCancelable(self): + """ + Wrapper around IProgress.cancelable. + """ + try: + fRc = self.o.cancelable; + except: + reporter.logXcpt(); + fRc = False; + return fRc; + + def wasCanceled(self): + """ + Wrapper around IProgress.canceled. + """ + try: + fRc = self.o.canceled; + except: + reporter.logXcpt(self.sName); + fRc = False; + return fRc; + + def cancel(self): + """ + Wrapper around IProgress.cancel() + Returns True on success, False on failure (logged as error). + """ + try: + self.o.cancel(); + except: + reporter.errorXcpt(self.sName); + return False; + return True; + + def getResult(self): + """ + Wrapper around IProgress.resultCode. + """ + try: + iRc = self.o.resultCode; + except: + reporter.logXcpt(self.sName); + iRc = -1; + return iRc; + + def getErrInfoResultCode(self): + """ + Wrapper around IProgress.errorInfo.resultCode. + + Returns the string on success, -1 on bad objects (logged as error), and + -2 on missing errorInfo object. + """ + iRc = -1; + try: + oErrInfo = self.o.errorInfo; + except: + reporter.errorXcpt(self.sName); + else: + if oErrInfo is None: + iRc = -2; + else: + try: + iRc = oErrInfo.resultCode; + except: + reporter.errorXcpt(); + return iRc; + + def getErrInfoText(self): + """ + Wrapper around IProgress.errorInfo.text. + + Returns the string on success, None on failure. Missing errorInfo is + not logged as an error, all other failures are. + """ + sText = None; + try: + oErrInfo = self.o.errorInfo; + except: + reporter.log2Xcpt(self.sName); + else: + if oErrInfo is not None: + try: + sText = oErrInfo.text; + except: + reporter.errorXcpt(); + return sText; + + def stringifyErrorInfo(self): + """ + Formats IProgress.errorInfo into a string. + """ + try: + oErrInfo = self.o.errorInfo; + except: + reporter.logXcpt(self.sName); + sErr = 'no error info'; + else: + sErr = vbox.stringifyErrorInfo(oErrInfo); + return sErr; + + def stringifyResult(self): + """ + Stringify the result. + """ + if self.isCompleted(): + if self.wasCanceled(): + sRet = 'Progress %s: Canceled, hrc=%s' % (self.sName, vbox.ComError.toString(self.getResult())); + elif self.getResult() == 0: + sRet = 'Progress %s: Success' % (self.sName,); + elif self.getResult() > 0: + sRet = 'Progress %s: Success (hrc=%s)' % (self.sName, vbox.ComError.toString(self.getResult())); + else: + sRet = 'Progress %s: Failed! %s' % (self.sName, self.stringifyErrorInfo()); + else: + sRet = 'Progress %s: Not completed yet...' % (self.sName); + return sRet; + + def logResult(self, fIgnoreErrors = False): + """ + Logs the result, failure logged as error unless fIgnoreErrors is True. + Return True on success, False on failure (and fIgnoreErrors is false). + """ + sText = self.stringifyResult(); + if self.isCompleted() and self.getResult() < 0 and fIgnoreErrors is False: + return reporter.error(sText); + reporter.log(sText); + return True; + + def waitOnProgress(self, cMsInterval = 1000): + """ + See vbox.TestDriver.waitOnProgress. + """ + self.doQuickApiTest(); + return self.oTstDrv.waitOnProgress(self.o, cMsInterval); + + def wait(self, cMsTimeout = 60000, fErrorOnTimeout = True, cMsInterval = 1000): + """ + Wait on the progress object for a while. + + Returns the resultCode of the progress object if completed. + Returns -1 on timeout, logged as error if fErrorOnTimeout is set. + Returns -2 is the progress object is invalid or waitForCompletion + fails (logged as errors). + """ + msStart = base.timestampMilli(); + while True: + self.oTstDrv.processPendingEvents(); + self.doQuickApiTest(); + try: + if self.o.completed: + break; + except: + reporter.errorXcpt(self.sName); + return -2; + self.oTstDrv.processPendingEvents(); + + cMsElapsed = base.timestampMilli() - msStart; + if cMsElapsed > cMsTimeout: + if fErrorOnTimeout: + reporter.error('Timing out after waiting for %u s on "%s"' % (cMsTimeout / 1000, self.sName)) + return -1; + + try: + self.o.waitForCompletion(cMsInterval); + except: + reporter.errorXcpt(self.sName); + return -2; + reporter.doPollWork('ProgressWrapper.wait'); + + try: + rc = self.o.resultCode; + except: + rc = -2; + reporter.errorXcpt(self.sName); + self.oTstDrv.processPendingEvents(); + return rc; + + def waitForOperation(self, iOperation, cMsTimeout = 60000, fErrorOnTimeout = True, cMsInterval = 1000, \ + fIgnoreErrors = False): + """ + Wait for the completion of a operation. + + Negative iOperation values are relative to operationCount (this + property may changed at runtime). + + Returns 0 if the operation completed normally. + Returns -1 on timeout, logged as error if fErrorOnTimeout is set. + Returns -2 is the progress object is invalid or waitForCompletion + fails (logged as errors). + Returns -3 if if the operation completed with an error, this is logged + as an error. + """ + msStart = base.timestampMilli(); + while True: + self.oTstDrv.processPendingEvents(); + self.doQuickApiTest(); + try: + iCurrentOperation = self.o.operation; + cOperations = self.o.operationCount; + if iOperation >= 0: + iRealOperation = iOperation; + else: + iRealOperation = cOperations + iOperation; + + if iCurrentOperation > iRealOperation: + return 0; + if iCurrentOperation == iRealOperation \ + and iRealOperation >= cOperations - 1 \ + and self.o.completed: + if self.o.resultCode < 0: + self.logResult(fIgnoreErrors); + return -3; + return 0; + except: + if fIgnoreErrors: + reporter.logXcpt(); + else: + reporter.errorXcpt(); + return -2; + self.oTstDrv.processPendingEvents(); + + cMsElapsed = base.timestampMilli() - msStart; + if cMsElapsed > cMsTimeout: + if fErrorOnTimeout: + if fIgnoreErrors: + reporter.log('Timing out after waiting for %s s on "%s" operation %d' \ + % (cMsTimeout / 1000, self.sName, iOperation)) + else: + reporter.error('Timing out after waiting for %s s on "%s" operation %d' \ + % (cMsTimeout / 1000, self.sName, iOperation)) + return -1; + + try: + self.o.waitForOperationCompletion(iRealOperation, cMsInterval); + except: + if fIgnoreErrors: + reporter.logXcpt(self.sName); + else: + reporter.errorXcpt(self.sName); + return -2; + reporter.doPollWork('ProgressWrapper.waitForOperation'); + # Not reached. + return -3; # Make pylin happy (for now). + + def doQuickApiTest(self): + """ + Queries everything that is stable and easy to get at and checks that + they don't throw errors. + """ + if True is True: # pylint: disable=comparison-with-itself + try: + iPct = self.o.operationPercent; + sDesc = self.o.description; + fCancelable = self.o.cancelable; + cSecsRemain = self.o.timeRemaining; + fCanceled = self.o.canceled; + fCompleted = self.o.completed; + iOp = self.o.operation; + cOps = self.o.operationCount; + iOpPct = self.o.operationPercent; + sOpDesc = self.o.operationDescription; + except: + reporter.errorXcpt('%s: %s' % (self.sName, self.o,)); + return False; + try: + # Very noisy -- only enable for debugging purposes. + #reporter.log2('%s: op=%u/%u/%s: %u%%; total=%u%% cancel=%s/%s compl=%s rem=%us; desc=%s' \ + # % (self.sName, iOp, cOps, sOpDesc, iOpPct, iPct, fCanceled, fCancelable, fCompleted, \ + # cSecsRemain, sDesc)); + _ = iPct; _ = sDesc; _ = fCancelable; _ = cSecsRemain; _ = fCanceled; _ = fCompleted; _ = iOp; + _ = cOps; _ = iOpPct; _ = sOpDesc; + except: + reporter.errorXcpt(); + return False; + + return True; + + +class SessionWrapper(TdTaskBase): + """ + Wrapper around a machine session. The real session object can be accessed + thru the o member (short is good, right :-). + """ + + def __init__(self, oSession, oVM, oVBox, oVBoxMgr, oTstDrv, fRemoteSession, sFallbackName = None, sLogFile = None): + """ + Initializes the session wrapper. + """ + TdTaskBase.__init__(self, utils.getCallerName()); + self.o = oSession; + self.oVBox = oVBox; + self.oVBoxMgr = oVBoxMgr; + self.oVM = oVM; # Not the session machine. Useful backdoor... + self.oTstDrv = oTstDrv; + self.fpApiVer = oTstDrv.fpApiVer; + self.fRemoteSession = fRemoteSession; + self.sLogFile = sLogFile; + self.oConsoleEventHandler = None; + self.uPid = None; + self.fPidFile = True; + self.fHostMemoryLow = False; # see signalHostMemoryLow; read-only for outsiders. + + try: + self.sName = oSession.machine.name; + except: + if sFallbackName is not None: + self.sName = sFallbackName; + else: + try: self.sName = str(oSession.machine); + except: self.sName = 'is-this-vm-already-off' + + try: + self.sUuid = oSession.machine.id; + except: + self.sUuid = None; + + # Try cache the SessionPID. + self.getPid(); + + def __del__(self): + """ + Destructor that makes sure the callbacks are deregistered and + that the session is closed. + """ + self.deregisterEventHandlerForTask(); + + if self.o is not None: + try: + self.close(); + reporter.log('close session %s' % (self.o)); + except: + pass; + self.o = None; + + TdTaskBase.__del__(self); + + def toString(self): + return '<%s: sUuid=%s, sName=%s, uPid=%s, sDbgCreated=%s, fRemoteSession=%s, oSession=%s,' \ + ' oConsoleEventHandler=%s, oVM=%s >' \ + % (type(self).__name__, self.sUuid, self.sName, self.uPid, self.sDbgCreated, self.fRemoteSession, + self.o, self.oConsoleEventHandler, self.oVM,); + + def __str__(self): + return self.toString(); + + # + # TdTaskBase overrides. + # + + def __pollTask(self): + """ Internal poller """ + # Poll for events after doing the remote GetState call, otherwise we + # might end up sleepless because XPCOM queues a cleanup event. + try: + try: + eState = self.o.machine.state; + except Exception as oXcpt: + if vbox.ComError.notEqual(oXcpt, vbox.ComError.E_UNEXPECTED): + reporter.logXcpt(); + return True; + finally: + self.oTstDrv.processPendingEvents(); + + # Switch + if eState == vboxcon.MachineState_Running: + return False; + if eState == vboxcon.MachineState_Paused: + return False; + if eState == vboxcon.MachineState_Teleporting: + return False; + if eState == vboxcon.MachineState_LiveSnapshotting: + return False; + if eState == vboxcon.MachineState_Starting: + return False; + if eState == vboxcon.MachineState_Stopping: + return False; + if eState == vboxcon.MachineState_Saving: + return False; + if eState == vboxcon.MachineState_Restoring: + return False; + if eState == vboxcon.MachineState_TeleportingPausedVM: + return False; + if eState == vboxcon.MachineState_TeleportingIn: + return False; + + # *Beeep* fudge! + if self.fpApiVer < 3.2 \ + and eState == vboxcon.MachineState_PoweredOff \ + and self.getAgeAsMs() < 3000: + return False; + + reporter.log('SessionWrapper::pollTask: eState=%s' % (eState)); + return True; + + + def pollTask(self, fLocked = False): + """ + Overrides TdTaskBase.pollTask(). + + This method returns False while the VM is online and running normally. + """ + + # Call super to check if the task was signalled by runtime error or similar, + # if not then check the VM state via __pollTask. + fRc = super(SessionWrapper, self).pollTask(fLocked); + if not fRc: + fRc = self.__pollTask(); + + # HACK ALERT: Lazily try registering the console event handler if + # we're not ready. + if not fRc and self.oConsoleEventHandler is None: + self.registerEventHandlerForTask(); + + # HACK ALERT: Lazily try get the PID and add it to the PID file. + if not fRc and self.uPid is None: + self.getPid(); + + return fRc; + + def waitForTask(self, cMsTimeout = 0): + """ + Overrides TdTaskBase.waitForTask(). + Process XPCOM/COM events while waiting. + """ + msStart = base.timestampMilli(); + fState = self.pollTask(False); + while not fState: + cMsElapsed = base.timestampMilli() - msStart; + if cMsElapsed > cMsTimeout: + break; + cMsSleep = cMsTimeout - cMsElapsed; + cMsSleep = min(cMsSleep, 10000); + try: self.oVBoxMgr.waitForEvents(cMsSleep); + except KeyboardInterrupt: raise; + except: pass; + if self.fnProcessEvents: + self.fnProcessEvents(); + reporter.doPollWork('SessionWrapper.waitForTask'); + fState = self.pollTask(False); + return fState; + + def setTaskOwner(self, oOwner): + """ + HACK ALERT! + Overrides TdTaskBase.setTaskOwner() so we can try call + registerEventHandlerForTask() again when when the testdriver calls + addTask() after VM has been spawned. Related to pollTask() above. + + The testdriver must not add the task too early for this to work! + """ + if oOwner is not None: + self.registerEventHandlerForTask() + return TdTaskBase.setTaskOwner(self, oOwner); + + + # + # Task helpers. + # + + def registerEventHandlerForTask(self): + """ + Registers the console event handlers for working the task state. + """ + if self.oConsoleEventHandler is not None: + return True; + self.oConsoleEventHandler = self.registerDerivedEventHandler(vbox.SessionConsoleEventHandler, {}, False); + return self.oConsoleEventHandler is not None; + + def deregisterEventHandlerForTask(self): + """ + Deregisters the console event handlers. + """ + if self.oConsoleEventHandler is not None: + self.oConsoleEventHandler.unregister(); + self.oConsoleEventHandler = None; + + def signalHostMemoryLow(self): + """ + Used by a runtime error event handler to indicate that we're low on memory. + Signals the task. + """ + self.fHostMemoryLow = True; + self.signalTask(); + return True; + + def needsPoweringOff(self): + """ + Examins the machine state to see if the VM needs powering off. + """ + try: + try: + eState = self.o.machine.state; + except Exception as oXcpt: + if vbox.ComError.notEqual(oXcpt, vbox.ComError.E_UNEXPECTED): + reporter.logXcpt(); + return False; + finally: + self.oTstDrv.processPendingEvents(); + + # Switch + if eState == vboxcon.MachineState_Running: + return True; + if eState == vboxcon.MachineState_Paused: + return True; + if eState == vboxcon.MachineState_Stuck: + return True; + if eState == vboxcon.MachineState_Teleporting: + return True; + if eState == vboxcon.MachineState_LiveSnapshotting: + return True; + if eState == vboxcon.MachineState_Starting: + return True; + if eState == vboxcon.MachineState_Saving: + return True; + if eState == vboxcon.MachineState_Restoring: + return True; + if eState == vboxcon.MachineState_TeleportingPausedVM: + return True; + if eState == vboxcon.MachineState_TeleportingIn: + return True; + if hasattr(vboxcon, 'MachineState_FaultTolerantSyncing'): + if eState == vboxcon.MachineState_FaultTolerantSyncing: + return True; + return False; + + def assertPoweredOff(self): + """ + Asserts that the VM is powered off, reporting an error if not. + Returns True if powered off, False + error msg if not. + """ + try: + try: + eState = self.oVM.state; + except Exception: + reporter.errorXcpt(); + return True; + finally: + self.oTstDrv.processPendingEvents(); + + if eState == vboxcon.MachineState_PoweredOff: + return True; + reporter.error('Expected machine state "PoweredOff", machine is in the "%s" state instead.' + % (_nameMachineState(eState),)); + return False; + + def getMachineStateWithName(self): + """ + Gets the current machine state both as a constant number/whatever and + as a human readable string. On error, the constants will be set to + None and the string will be the error message. + """ + try: + eState = self.oVM.state; + except: + return (None, '[error getting state: %s]' % (self.oVBoxMgr.xcptToString(),)); + finally: + self.oTstDrv.processPendingEvents(); + return (eState, _nameMachineState(eState)); + + def reportPrematureTermination(self, sPrefix = ''): + """ + Reports a premature virtual machine termination. + Returns False to facilitate simpler error paths. + """ + + reporter.error(sPrefix + 'The virtual machine terminated prematurely!!'); + (enmState, sStateNm) = self.getMachineStateWithName(); + reporter.error(sPrefix + 'Machine state: %s' % (sStateNm,)); + + if enmState is not None \ + and enmState == vboxcon.MachineState_Aborted \ + and self.uPid is not None: + # + # Look for process crash info. + # + def addCrashFile(sLogFile, fBinary): + """ processCollectCrashInfo callback. """ + reporter.addLogFile(sLogFile, 'crash/dump/vm' if fBinary else 'crash/report/vm'); + utils.processCollectCrashInfo(self.uPid, reporter.log, addCrashFile); + + return False; + + + + # + # ISession / IMachine / ISomethingOrAnother wrappers. + # + + def close(self): + """ + Closes the session if it's open and removes it from the + vbox.TestDriver.aoRemoteSessions list. + Returns success indicator. + """ + fRc = True; + if self.o is not None: + # Get the pid in case we need to kill the process later on. + self.getPid(); + + # Try close it. + try: + if self.fpApiVer < 3.3: + self.o.close(); + else: + self.o.unlockMachine(); + self.o = None; + except KeyboardInterrupt: + raise; + except: + # Kludge to ignore VBoxSVC's closing of our session when the + # direct session closes / VM process terminates. Fun! + try: fIgnore = self.o.state == vboxcon.SessionState_Unlocked; + except: fIgnore = False; + if fIgnore: + self.o = None; # Must prevent a retry during GC. + else: + reporter.errorXcpt('ISession::unlockMachine failed on %s' % (self.o)); + fRc = False; + + # Remove it from the remote session list if applicable (not 100% clean). + if fRc and self.fRemoteSession: + try: + if self in self.oTstDrv.aoRemoteSessions: + reporter.log2('SessionWrapper::close: Removing myself from oTstDrv.aoRemoteSessions'); + self.oTstDrv.aoRemoteSessions.remove(self) + except: + reporter.logXcpt(); + + if self.uPid is not None and self.fPidFile: + self.oTstDrv.pidFileRemove(self.uPid); + self.fPidFile = False; + + # It's only logical to deregister the event handler after the session + # is closed. It also avoids circular references between the session + # and the listener, which causes trouble with garbage collection. + self.deregisterEventHandlerForTask(); + + self.oTstDrv.processPendingEvents(); + return fRc; + + def saveSettings(self, fClose = False): + """ + Saves the settings and optionally closes the session. + Returns success indicator. + """ + try: + try: + self.o.machine.saveSettings(); + except: + reporter.errorXcpt('saveSettings failed on %s' % (self.o)); + return False; + finally: + self.oTstDrv.processPendingEvents(); + if fClose: + return self.close(); + return True; + + def discardSettings(self, fClose = False): + """ + Discards the settings and optionally closes the session. + """ + try: + try: + self.o.machine.discardSettings(); + except: + reporter.errorXcpt('discardSettings failed on %s' % (self.o)); + return False; + finally: + self.oTstDrv.processPendingEvents(); + if fClose: + return self.close(); + return True; + + def enableVirtEx(self, fEnable): + """ + Enables or disables AMD-V/VT-x. + Returns True on success and False on failure. Error information is logged. + """ + # Enable/disable it. + fRc = True; + try: + self.o.machine.setHWVirtExProperty(vboxcon.HWVirtExPropertyType_Enabled, fEnable); + except: + reporter.errorXcpt('failed to set HWVirtExPropertyType_Enabled=%s for "%s"' % (fEnable, self.sName)); + fRc = False; + else: + reporter.log('set HWVirtExPropertyType_Enabled=%s for "%s"' % (fEnable, self.sName)); + + # Force/unforce it. + if fRc and hasattr(vboxcon, 'HWVirtExPropertyType_Force'): + try: + self.o.machine.setHWVirtExProperty(vboxcon.HWVirtExPropertyType_Force, fEnable); + except: + reporter.errorXcpt('failed to set HWVirtExPropertyType_Force=%s for "%s"' % (fEnable, self.sName)); + fRc = False; + else: + reporter.log('set HWVirtExPropertyType_Force=%s for "%s"' % (fEnable, self.sName)); + else: + reporter.log('Warning! vboxcon has no HWVirtExPropertyType_Force attribute.'); + ## @todo Modify CFGM to do the same for old VBox versions? + + self.oTstDrv.processPendingEvents(); + return fRc; + + def enableNestedPaging(self, fEnable): + """ + Enables or disables nested paging.. + Returns True on success and False on failure. Error information is logged. + """ + ## @todo Add/remove force CFGM thing, we don't want fallback logic when testing. + fRc = True; + try: + self.o.machine.setHWVirtExProperty(vboxcon.HWVirtExPropertyType_NestedPaging, fEnable); + except: + reporter.errorXcpt('failed to set HWVirtExPropertyType_NestedPaging=%s for "%s"' % (fEnable, self.sName)); + fRc = False; + else: + reporter.log('set HWVirtExPropertyType_NestedPaging=%s for "%s"' % (fEnable, self.sName)); + self.oTstDrv.processPendingEvents(); + return fRc; + + def enableLongMode(self, fEnable): + """ + Enables or disables LongMode. + Returns True on success and False on failure. Error information is logged. + """ + # Supported. + if self.fpApiVer < 4.2 or not hasattr(vboxcon, 'HWVirtExPropertyType_LongMode'): + return True; + + # Enable/disable it. + fRc = True; + try: + self.o.machine.setCPUProperty(vboxcon.CPUPropertyType_LongMode, fEnable); + except: + reporter.errorXcpt('failed to set CPUPropertyType_LongMode=%s for "%s"' % (fEnable, self.sName)); + fRc = False; + else: + reporter.log('set CPUPropertyType_LongMode=%s for "%s"' % (fEnable, self.sName)); + self.oTstDrv.processPendingEvents(); + return fRc; + + def enableNestedHwVirt(self, fEnable): + """ + Enables or disables Nested Hardware-Virtualization. + Returns True on success and False on failure. Error information is logged. + """ + # Supported. + if self.fpApiVer < 5.3 or not hasattr(vboxcon, 'CPUPropertyType_HWVirt'): + return True; + + # Enable/disable it. + fRc = True; + try: + self.o.machine.setCPUProperty(vboxcon.CPUPropertyType_HWVirt, fEnable); + except: + reporter.errorXcpt('failed to set CPUPropertyType_HWVirt=%s for "%s"' % (fEnable, self.sName)); + fRc = False; + else: + reporter.log('set CPUPropertyType_HWVirt=%s for "%s"' % (fEnable, self.sName)); + self.oTstDrv.processPendingEvents(); + return fRc; + + def enablePae(self, fEnable): + """ + Enables or disables PAE + Returns True on success and False on failure. Error information is logged. + """ + fRc = True; + try: + if self.fpApiVer >= 3.2: # great, ain't it? + self.o.machine.setCPUProperty(vboxcon.CPUPropertyType_PAE, fEnable); + else: + self.o.machine.setCpuProperty(vboxcon.CpuPropertyType_PAE, fEnable); + except: + reporter.errorXcpt('failed to set CPUPropertyType_PAE=%s for "%s"' % (fEnable, self.sName)); + fRc = False; + else: + reporter.log('set CPUPropertyType_PAE=%s for "%s"' % (fEnable, self.sName)); + self.oTstDrv.processPendingEvents(); + return fRc; + + def enableIoApic(self, fEnable): + """ + Enables or disables the IO-APIC + Returns True on success and False on failure. Error information is logged. + """ + fRc = True; + try: + self.o.machine.BIOSSettings.IOAPICEnabled = fEnable; + except: + reporter.errorXcpt('failed to set BIOSSettings.IOAPICEnabled=%s for "%s"' % (fEnable, self.sName)); + fRc = False; + else: + reporter.log('set BIOSSettings.IOAPICEnabled=%s for "%s"' % (fEnable, self.sName)); + self.oTstDrv.processPendingEvents(); + return fRc; + + def enableHpet(self, fEnable): + """ + Enables or disables the HPET + Returns True on success and False on failure. Error information is logged. + """ + fRc = True; + try: + if self.fpApiVer >= 4.2: + self.o.machine.HPETEnabled = fEnable; + else: + self.o.machine.hpetEnabled = fEnable; + except: + reporter.errorXcpt('failed to set HpetEnabled=%s for "%s"' % (fEnable, self.sName)); + fRc = False; + else: + reporter.log('set HpetEnabled=%s for "%s"' % (fEnable, self.sName)); + self.oTstDrv.processPendingEvents(); + return fRc; + + def enableUsbHid(self, fEnable): + """ + Enables or disables the USB HID + Returns True on success and False on failure. Error information is logged. + """ + fRc = True; + try: + if fEnable: + if self.fpApiVer >= 4.3: + cOhciCtls = self.o.machine.getUSBControllerCountByType(vboxcon.USBControllerType_OHCI); + if cOhciCtls == 0: + self.o.machine.addUSBController('OHCI', vboxcon.USBControllerType_OHCI); + else: + self.o.machine.usbController.enabled = True; + + if self.fpApiVer >= 4.2: + self.o.machine.pointingHIDType = vboxcon.PointingHIDType_ComboMouse; + self.o.machine.keyboardHIDType = vboxcon.KeyboardHIDType_ComboKeyboard; + else: + self.o.machine.pointingHidType = vboxcon.PointingHidType_ComboMouse; + self.o.machine.keyboardHidType = vboxcon.KeyboardHidType_ComboKeyboard; + else: + if self.fpApiVer >= 4.2: + self.o.machine.pointingHIDType = vboxcon.PointingHIDType_PS2Mouse; + self.o.machine.keyboardHIDType = vboxcon.KeyboardHIDType_PS2Keyboard; + else: + self.o.machine.pointingHidType = vboxcon.PointingHidType_PS2Mouse; + self.o.machine.keyboardHidType = vboxcon.KeyboardHidType_PS2Keyboard; + except: + reporter.errorXcpt('failed to change UsbHid to %s for "%s"' % (fEnable, self.sName)); + fRc = False; + else: + reporter.log('changed UsbHid to %s for "%s"' % (fEnable, self.sName)); + self.oTstDrv.processPendingEvents(); + return fRc; + + def enableUsbOhci(self, fEnable): + """ + Enables or disables the USB OHCI controller + Returns True on success and False on failure. Error information is logged. + """ + fRc = True; + try: + if fEnable: + if self.fpApiVer >= 4.3: + cOhciCtls = self.o.machine.getUSBControllerCountByType(vboxcon.USBControllerType_OHCI); + if cOhciCtls == 0: + self.o.machine.addUSBController('OHCI', vboxcon.USBControllerType_OHCI); + else: + self.o.machine.usbController.enabled = True; + else: + if self.fpApiVer >= 4.3: + cOhciCtls = self.o.machine.getUSBControllerCountByType(vboxcon.USBControllerType_OHCI); + if cOhciCtls == 1: + self.o.machine.removeUSBController('OHCI'); + else: + self.o.machine.usbController.enabled = False; + except: + reporter.errorXcpt('failed to change OHCI to %s for "%s"' % (fEnable, self.sName)); + fRc = False; + else: + reporter.log('changed OHCI to %s for "%s"' % (fEnable, self.sName)); + self.oTstDrv.processPendingEvents(); + return fRc; + + def enableUsbEhci(self, fEnable): + """ + Enables or disables the USB EHCI controller, enables also OHCI if it is still disabled. + Returns True on success and False on failure. Error information is logged. + """ + fRc = True; + try: + if fEnable: + if self.fpApiVer >= 4.3: + cOhciCtls = self.o.machine.getUSBControllerCountByType(vboxcon.USBControllerType_OHCI); + if cOhciCtls == 0: + self.o.machine.addUSBController('OHCI', vboxcon.USBControllerType_OHCI); + + cEhciCtls = self.o.machine.getUSBControllerCountByType(vboxcon.USBControllerType_EHCI); + if cEhciCtls == 0: + self.o.machine.addUSBController('EHCI', vboxcon.USBControllerType_EHCI); + else: + self.o.machine.usbController.enabled = True; + self.o.machine.usbController.enabledEHCI = True; + else: + if self.fpApiVer >= 4.3: + cEhciCtls = self.o.machine.getUSBControllerCountByType(vboxcon.USBControllerType_EHCI); + if cEhciCtls == 1: + self.o.machine.removeUSBController('EHCI'); + else: + self.o.machine.usbController.enabledEHCI = False; + except: + reporter.errorXcpt('failed to change EHCI to %s for "%s"' % (fEnable, self.sName)); + fRc = False; + else: + reporter.log('changed EHCI to %s for "%s"' % (fEnable, self.sName)); + self.oTstDrv.processPendingEvents(); + return fRc; + + def enableUsbXhci(self, fEnable): + """ + Enables or disables the USB XHCI controller. Error information is logged. + """ + fRc = True; + try: + if fEnable: + cXhciCtls = self.o.machine.getUSBControllerCountByType(vboxcon.USBControllerType_XHCI); + if cXhciCtls == 0: + self.o.machine.addUSBController('XHCI', vboxcon.USBControllerType_XHCI); + else: + cXhciCtls = self.o.machine.getUSBControllerCountByType(vboxcon.USBControllerType_XHCI); + if cXhciCtls == 1: + self.o.machine.removeUSBController('XHCI'); + except: + reporter.errorXcpt('failed to change XHCI to %s for "%s"' % (fEnable, self.sName)); + fRc = False; + else: + reporter.log('changed XHCI to %s for "%s"' % (fEnable, self.sName)); + self.oTstDrv.processPendingEvents(); + return fRc; + + def setFirmwareType(self, eType): + """ + Sets the firmware type. + Returns True on success and False on failure. Error information is logged. + """ + fRc = True; + try: + self.o.machine.firmwareType = eType; + except: + reporter.errorXcpt('failed to set firmwareType=%s for "%s"' % (eType, self.sName)); + fRc = False; + else: + reporter.log('set firmwareType=%s for "%s"' % (eType, self.sName)); + self.oTstDrv.processPendingEvents(); + return fRc; + + def setChipsetType(self, eType): + """ + Sets the chipset type. + Returns True on success and False on failure. Error information is logged. + """ + fRc = True; + try: + self.o.machine.chipsetType = eType; + except: + reporter.errorXcpt('failed to set chipsetType=%s for "%s"' % (eType, self.sName)); + fRc = False; + else: + reporter.log('set chipsetType=%s for "%s"' % (eType, self.sName)); + self.oTstDrv.processPendingEvents(); + return fRc; + + def setIommuType(self, eType): + """ + Sets the IOMMU type. + Returns True on success and False on failure. Error information is logged. + """ + # Supported. + if self.fpApiVer < 6.2 or not hasattr(vboxcon, 'IommuType_Intel') or not hasattr(vboxcon, 'IommuType_AMD'): + return True; + fRc = True; + try: + self.o.machine.iommuType = eType; + except: + reporter.errorXcpt('failed to set iommuType=%s for "%s"' % (eType, self.sName)); + fRc = False; + else: + reporter.log('set iommuType=%s for "%s"' % (eType, self.sName)); + self.oTstDrv.processPendingEvents(); + return fRc; + + def setupBootLogo(self, fEnable, cMsLogoDisplay = 0): + """ + Sets up the boot logo. fEnable toggles the fade and boot menu + settings as well as the mode. + """ + fRc = True; + try: + self.o.machine.BIOSSettings.logoFadeIn = not fEnable; + self.o.machine.BIOSSettings.logoFadeOut = not fEnable; + self.o.machine.BIOSSettings.logoDisplayTime = cMsLogoDisplay; + if fEnable: + self.o.machine.BIOSSettings.bootMenuMode = vboxcon.BIOSBootMenuMode_Disabled; + else: + self.o.machine.BIOSSettings.bootMenuMode = vboxcon.BIOSBootMenuMode_MessageAndMenu; + except: + reporter.errorXcpt('failed to set logoFadeIn/logoFadeOut/bootMenuMode=%s for "%s"' % (fEnable, self.sName)); + fRc = False; + else: + reporter.log('set logoFadeIn/logoFadeOut/bootMenuMode=%s for "%s"' % (fEnable, self.sName)); + self.oTstDrv.processPendingEvents(); + return fRc; + + def setupVrdp(self, fEnable, uPort = None): + """ + Configures VRDP. + """ + fRc = True; + try: + if self.fpApiVer >= 4.0: + self.o.machine.VRDEServer.enabled = fEnable; + else: + self.o.machine.VRDPServer.enabled = fEnable; + except: + reporter.errorXcpt('failed to set VRDEServer::enabled=%s for "%s"' % (fEnable, self.sName)); + fRc = False; + + if uPort is not None and fRc: + try: + if self.fpApiVer >= 4.0: + self.o.machine.VRDEServer.setVRDEProperty("TCP/Ports", str(uPort)); + else: + self.o.machine.VRDPServer.ports = str(uPort); + except: + reporter.errorXcpt('failed to set VRDEServer::ports=%s for "%s"' % (uPort, self.sName)); + fRc = False; + if fRc: + reporter.log('set VRDEServer.enabled/ports=%s/%s for "%s"' % (fEnable, uPort, self.sName)); + self.oTstDrv.processPendingEvents(); + return fRc; + + def getNicDriverNameFromType(self, eNicType): + """ + Helper that translate the adapter type into a driver name. + """ + if eNicType in (vboxcon.NetworkAdapterType_Am79C970A, vboxcon.NetworkAdapterType_Am79C973): + sName = 'pcnet'; + elif eNicType in (vboxcon.NetworkAdapterType_I82540EM, + vboxcon.NetworkAdapterType_I82543GC, + vboxcon.NetworkAdapterType_I82545EM): + sName = 'e1000'; + elif eNicType == vboxcon.NetworkAdapterType_Virtio: + sName = 'virtio-net'; + else: + reporter.error('Unknown adapter type "%s" (VM: "%s")' % (eNicType, self.sName)); + sName = 'pcnet'; + return sName; + + def setupNatForwardingForTxs(self, iNic = 0, iHostPort = 5042): + """ + Sets up NAT forwarding for port 5042 if applicable, cleans up if not. + """ + try: + oNic = self.o.machine.getNetworkAdapter(iNic); + except: + reporter.errorXcpt('getNetworkAdapter(%s) failed for "%s"' % (iNic, self.sName)); + return False; + + # Nuke the old setup for all possible adapter types (in case we're + # called after it changed). + for sName in ('pcnet', 'e1000', 'virtio-net'): + for sConfig in ('VBoxInternal/Devices/%s/%u/LUN#0/AttachedDriver/Config' % (sName, iNic), \ + 'VBoxInternal/Devices/%s/%u/LUN#0/Config' % (sName, iNic)): + try: + self.o.machine.setExtraData('%s/txs/Protocol' % (sConfig), ''); + self.o.machine.setExtraData('%s/txs/HostPort' % (sConfig), ''); + self.o.machine.setExtraData('%s/txs/GuestPort' % (sConfig), ''); + except: + reporter.errorXcpt(); + + # Set up port forwarding if NAT attachment. + try: + eAttType = oNic.attachmentType; + except: + reporter.errorXcpt('attachmentType on %s failed for "%s"' % (iNic, self.sName)); + return False; + if eAttType != vboxcon.NetworkAttachmentType_NAT: + return True; + + try: + eNicType = oNic.adapterType; + fTraceEnabled = oNic.traceEnabled; + except: + reporter.errorXcpt('attachmentType/traceEnabled on %s failed for "%s"' % (iNic, self.sName)); + return False; + + if self.fpApiVer >= 4.1: + try: + if self.fpApiVer >= 4.2: + oNatEngine = oNic.NATEngine; + else: + oNatEngine = oNic.natDriver; + except: + reporter.errorXcpt('Failed to get INATEngine data on "%s"' % (self.sName)); + return False; + try: oNatEngine.removeRedirect('txs'); + except: pass; + try: + oNatEngine.addRedirect('txs', vboxcon.NATProtocol_TCP, '127.0.0.1', '%s' % (iHostPort), '', '5042'); + except: + reporter.errorXcpt('Failed to add a addRedirect redirect on "%s"' % (self.sName)); + return False; + + else: + sName = self.getNicDriverNameFromType(eNicType); + if fTraceEnabled: + sConfig = 'VBoxInternal/Devices/%s/%u/LUN#0/AttachedDriver/Config' % (sName, iNic) + else: + sConfig = 'VBoxInternal/Devices/%s/%u/LUN#0/Config' % (sName, iNic) + + try: + self.o.machine.setExtraData('%s/txs/Protocol' % (sConfig), 'TCP'); + self.o.machine.setExtraData('%s/txs/HostPort' % (sConfig), '%s' % (iHostPort)); + self.o.machine.setExtraData('%s/txs/GuestPort' % (sConfig), '5042'); + except: + reporter.errorXcpt('Failed to set NAT extra data on "%s"' % (self.sName)); + return False; + return True; + + def setNicType(self, eType, iNic = 0): + """ + Sets the NIC type of the specified NIC. + Returns True on success and False on failure. Error information is logged. + """ + try: + try: + oNic = self.o.machine.getNetworkAdapter(iNic); + except: + reporter.errorXcpt('getNetworkAdapter(%s) failed for "%s"' % (iNic, self.sName)); + return False; + try: + oNic.adapterType = eType; + except: + reporter.errorXcpt('failed to set NIC type on slot %s to %s for VM "%s"' % (iNic, eType, self.sName)); + return False; + finally: + self.oTstDrv.processPendingEvents(); + + if not self.setupNatForwardingForTxs(iNic): + return False; + reporter.log('set NIC type on slot %s to %s for VM "%s"' % (iNic, eType, self.sName)); + return True; + + def setNicTraceEnabled(self, fTraceEnabled, sTraceFile, iNic = 0): + """ + Sets the NIC trace enabled flag and file path. + Returns True on success and False on failure. Error information is logged. + """ + try: + try: + oNic = self.o.machine.getNetworkAdapter(iNic); + except: + reporter.errorXcpt('getNetworkAdapter(%s) failed for "%s"' % (iNic, self.sName)); + return False; + try: + oNic.traceEnabled = fTraceEnabled; + oNic.traceFile = sTraceFile; + except: + reporter.errorXcpt('failed to set NIC trace flag on slot %s to %s for VM "%s"' \ + % (iNic, fTraceEnabled, self.sName)); + return False; + finally: + self.oTstDrv.processPendingEvents(); + + if not self.setupNatForwardingForTxs(iNic): + return False; + reporter.log('set NIC trace on slot %s to "%s" (path "%s") for VM "%s"' % + (iNic, fTraceEnabled, sTraceFile, self.sName)); + return True; + + def getDefaultNicName(self, eAttachmentType): + """ + Return the default network / interface name for the NIC attachment type. + """ + sRetName = ''; + if eAttachmentType == vboxcon.NetworkAttachmentType_Bridged: + if self.oTstDrv.sDefBridgedNic is not None: + sRetName = self.oTstDrv.sDefBridgedNic; + else: + sRetName = 'eth0'; + try: + aoHostNics = self.oVBoxMgr.getArray(self.oVBox.host, 'networkInterfaces'); + for oHostNic in aoHostNics: + if oHostNic.interfaceType == vboxcon.HostNetworkInterfaceType_Bridged \ + and oHostNic.status == vboxcon.HostNetworkInterfaceStatus_Up: + sRetName = oHostNic.name; + break; + except: + reporter.errorXcpt(); + + elif eAttachmentType == vboxcon.NetworkAttachmentType_HostOnly: + try: + aoHostNics = self.oVBoxMgr.getArray(self.oVBox.host, 'networkInterfaces'); + for oHostNic in aoHostNics: + if oHostNic.interfaceType == vboxcon.HostNetworkInterfaceType_HostOnly: + if oHostNic.status == vboxcon.HostNetworkInterfaceStatus_Up: + sRetName = oHostNic.name; + break; + if sRetName == '': + sRetName = oHostNic.name; + except: + reporter.errorXcpt(); + if sRetName == '': + # Create a new host-only interface. + reporter.log("Creating host only NIC ..."); + try: + (oIProgress, oIHostOnly) = self.oVBox.host.createHostOnlyNetworkInterface(); + oProgress = ProgressWrapper(oIProgress, self.oVBoxMgr, self.oTstDrv, 'Create host only NIC'); + oProgress.wait(); + if oProgress.logResult() is False: + return ''; + sRetName = oIHostOnly.name; + except: + reporter.errorXcpt(); + return ''; + reporter.log("Created host only NIC: '%s'" % (sRetName,)); + + elif self.fpApiVer >= 7.0 and eAttachmentType == vboxcon.NetworkAttachmentType_HostOnlyNetwork: + aoHostNetworks = self.oVBoxMgr.getArray(self.oVBox, 'hostOnlyNetworks'); + if aoHostNetworks: + sRetName = aoHostNetworks[0].networkName; + else: + try: + oHostOnlyNet = self.oVBox.createHostOnlyNetwork('Host-only Test Network'); + oHostOnlyNet.lowerIP = '192.168.56.1'; + oHostOnlyNet.upperIP = '192.168.56.199'; + oHostOnlyNet.networkMask = '255.255.255.0'; + sRetName = oHostOnlyNet.networkName; + except: + reporter.errorXcpt(); + return ''; + + elif eAttachmentType == vboxcon.NetworkAttachmentType_Internal: + sRetName = 'VBoxTest'; + + elif eAttachmentType == vboxcon.NetworkAttachmentType_NAT: + sRetName = ''; + + else: ## @todo Support NetworkAttachmentType_NATNetwork + reporter.error('eAttachmentType=%s is not known' % (eAttachmentType)); + return sRetName; + + def setNicAttachment(self, eAttachmentType, sName = None, iNic = 0): + """ + Sets the attachment type of the specified NIC. + Returns True on success and False on failure. Error information is logged. + """ + try: + oNic = self.o.machine.getNetworkAdapter(iNic); + except: + reporter.errorXcpt('getNetworkAdapter(%s) failed for "%s"' % (iNic, self.sName)); + return False; + + try: + if eAttachmentType is not None: + try: + if self.fpApiVer >= 4.1: + oNic.attachmentType = eAttachmentType; + else: + if eAttachmentType == vboxcon.NetworkAttachmentType_NAT: + oNic.attachToNAT(); + elif eAttachmentType == vboxcon.NetworkAttachmentType_Bridged: + oNic.attachToBridgedInterface(); + elif eAttachmentType == vboxcon.NetworkAttachmentType_Internal: + oNic.attachToInternalNetwork(); + elif eAttachmentType == vboxcon.NetworkAttachmentType_HostOnly: + oNic.attachToHostOnlyInterface(); + else: + raise base.GenError("eAttachmentType=%s is invalid" % (eAttachmentType)); + except: + reporter.errorXcpt('failed to set the attachment type on slot %s to %s for VM "%s"' \ + % (iNic, eAttachmentType, self.sName)); + return False; + else: + try: + eAttachmentType = oNic.attachmentType; + except: + reporter.errorXcpt('failed to get the attachment type on slot %s for VM "%s"' % (iNic, self.sName)); + return False; + finally: + self.oTstDrv.processPendingEvents(); + + if sName is not None: + # Resolve the special 'default' name. + if sName == 'default': + sName = self.getDefaultNicName(eAttachmentType); + + # The name translate to different attributes depending on the + # attachment type. + try: + if eAttachmentType == vboxcon.NetworkAttachmentType_Bridged: + ## @todo check this out on windows, may have to do a + # translation of the name there or smth IIRC. + try: + if self.fpApiVer >= 4.1: + oNic.bridgedInterface = sName; + else: + oNic.hostInterface = sName; + except: + reporter.errorXcpt('failed to set the hostInterface property on slot %s to "%s" for VM "%s"' + % (iNic, sName, self.sName,)); + return False; + elif eAttachmentType == vboxcon.NetworkAttachmentType_HostOnly: + try: + if self.fpApiVer >= 4.1: + oNic.hostOnlyInterface = sName; + else: + oNic.hostInterface = sName; + except: + reporter.errorXcpt('failed to set the internalNetwork property on slot %s to "%s" for VM "%s"' + % (iNic, sName, self.sName,)); + return False; + elif self.fpApiVer >= 7.0 and eAttachmentType == vboxcon.NetworkAttachmentType_HostOnlyNetwork: + try: + oNic.hostOnlyNetwork = sName; + except: + reporter.errorXcpt('failed to set the hostOnlyNetwork property on slot %s to "%s" for VM "%s"' + % (iNic, sName, self.sName,)); + return False; + elif eAttachmentType == vboxcon.NetworkAttachmentType_Internal: + try: + oNic.internalNetwork = sName; + except: + reporter.errorXcpt('failed to set the internalNetwork property on slot %s to "%s" for VM "%s"' + % (iNic, sName, self.sName,)); + return False; + elif eAttachmentType == vboxcon.NetworkAttachmentType_NAT: + try: + oNic.NATNetwork = sName; + except: + reporter.errorXcpt('failed to set the NATNetwork property on slot %s to "%s" for VM "%s"' + % (iNic, sName, self.sName,)); + return False; + finally: + self.oTstDrv.processPendingEvents(); + + if not self.setupNatForwardingForTxs(iNic): + return False; + reporter.log('set NIC attachment type on slot %s to %s for VM "%s"' % (iNic, eAttachmentType, self.sName)); + return True; + + def setNicLocalhostReachable(self, fReachable, iNic = 0): + """ + Sets whether the specified NIC can reach the host or not. + Only affects (enabled) NICs configured to NAT at the moment. + + Returns True on success and False on failure. Error information is logged. + """ + try: + oNic = self.o.machine.getNetworkAdapter(iNic); + except: + return reporter.errorXcpt('getNetworkAdapter(%s) failed for "%s"' % (iNic, self.sName,)); + + try: + if not oNic.enabled: # NIC not enabled? Nothing to do here. + return True; + except: + return reporter.errorXcpt('NIC enabled status (%s) failed for "%s"' % (iNic, self.sName,)); + + reporter.log('Setting "LocalhostReachable" for network adapter in slot %d to %s' % (iNic, fReachable)); + + try: + oNatEngine = oNic.NATEngine; + except: + return reporter.errorXcpt('Getting NIC NAT engine (%s) failed for "%s"' % (iNic, self.sName,)); + + try: + if hasattr(oNatEngine, "localhostReachable"): + oNatEngine.localhostReachable = fReachable; + else: + oNatEngine.LocalhostReachable = fReachable; + except: + return reporter.errorXcpt('LocalhostReachable (%s) failed for "%s"' % (iNic, self.sName,)); + + return True; + + def setNicMacAddress(self, sMacAddr, iNic = 0): + """ + Sets the MAC address of the specified NIC. + + The sMacAddr parameter is a string supplying the tail end of the MAC + address, missing quads are supplied from a constant byte (2), the IPv4 + address of the host, and the NIC number. + + Returns True on success and False on failure. Error information is logged. + """ + + # Resolve missing MAC address prefix by feeding in the host IP address bytes. + cchMacAddr = len(sMacAddr); + if 0 < cchMacAddr < 12: + sHostIP = netutils.getPrimaryHostIp(); + abHostIP = socket.inet_aton(sHostIP); + if sys.version_info[0] < 3: + abHostIP = (ord(abHostIP[0]), ord(abHostIP[1]), ord(abHostIP[2]), ord(abHostIP[3])); + + if abHostIP[0] == 127 \ + or (abHostIP[0] == 169 and abHostIP[1] == 254) \ + or (abHostIP[0] == 192 and abHostIP[1] == 168 and abHostIP[2] == 56): + return reporter.error('host IP for "%s" is %s, most likely not unique.' % (netutils.getHostnameFqdn(), sHostIP,)); + + sDefaultMac = '%02X%02X%02X%02X%02X%02X' % (0x02, abHostIP[0], abHostIP[1], abHostIP[2], abHostIP[3], iNic); + sMacAddr = sDefaultMac[0:(12 - cchMacAddr)] + sMacAddr; + + # Get the NIC object and try set it address. + try: + oNic = self.o.machine.getNetworkAdapter(iNic); + except: + return reporter.errorXcpt('getNetworkAdapter(%s) failed for "%s"' % (iNic, self.sName,)); + + try: + oNic.MACAddress = sMacAddr; + except: + return reporter.errorXcpt('failed to set the MAC address on slot %s to "%s" for VM "%s"' + % (iNic, sMacAddr, self.sName)); + + reporter.log('set MAC address on slot %s to %s for VM "%s"' % (iNic, sMacAddr, self.sName,)); + return True; + + def setRamSize(self, cMB): + """ + Set the RAM size of the VM. + Returns True on success and False on failure. Error information is logged. + """ + fRc = True; + try: + self.o.machine.memorySize = cMB; + except: + reporter.errorXcpt('failed to set the RAM size of "%s" to %s' % (self.sName, cMB)); + fRc = False; + else: + reporter.log('set the RAM size of "%s" to %s' % (self.sName, cMB)); + self.oTstDrv.processPendingEvents(); + return fRc; + + def setLargePages(self, fUseLargePages): + """ + Configures whether the VM should use large pages or not. + Returns True on success and False on failure. Error information is logged. + """ + fRc = True; + try: + self.o.machine.setHWVirtExProperty(vboxcon.HWVirtExPropertyType_LargePages, fUseLargePages); + except: + reporter.errorXcpt('failed to set large pages of "%s" to %s' % (self.sName, fUseLargePages)); + fRc = False; + else: + reporter.log('set the large pages of "%s" to %s' % (self.sName, fUseLargePages)); + self.oTstDrv.processPendingEvents(); + return fRc; + + def setVRamSize(self, cMB): + """ + Set the RAM size of the VM. + Returns True on success and False on failure. Error information is logged. + """ + fRc = True; + try: + if self.fpApiVer >= 6.1 and hasattr(self.o.machine, 'graphicsAdapter'): + self.o.machine.graphicsAdapter.VRAMSize = cMB; + else: + self.o.machine.VRAMSize = cMB; + except: + reporter.errorXcpt('failed to set the VRAM size of "%s" to %s' % (self.sName, cMB)); + fRc = False; + else: + reporter.log('set the VRAM size of "%s" to %s' % (self.sName, cMB)); + self.oTstDrv.processPendingEvents(); + return fRc; + + def setVideoControllerType(self, eControllerType): + """ + Set the video controller type of the VM. + Returns True on success and False on failure. Error information is logged. + """ + fRc = True; + try: + if self.fpApiVer >= 6.1 and hasattr(self.o.machine, 'graphicsAdapter'): + self.o.machine.graphicsAdapter.graphicsControllerType = eControllerType; + else: + self.o.machine.graphicsControllerType = eControllerType; + except: + reporter.errorXcpt('failed to set the video controller type of "%s" to %s' % (self.sName, eControllerType)); + fRc = False; + else: + reporter.log('set the video controller type of "%s" to %s' % (self.sName, eControllerType)); + self.oTstDrv.processPendingEvents(); + return fRc; + + def setAccelerate3DEnabled(self, fEnabled): + """ + Set the video controller type of the VM. + Returns True on success and False on failure. Error information is logged. + """ + fRc = True; + try: + if self.fpApiVer >= 6.1 and hasattr(self.o.machine, 'graphicsAdapter'): + self.o.machine.graphicsAdapter.accelerate3DEnabled = fEnabled; + else: + self.o.machine.accelerate3DEnabled = fEnabled; + except: + reporter.errorXcpt('failed to set the accelerate3DEnabled of "%s" to %s' % (self.sName, fEnabled)); + fRc = False; + else: + reporter.log('set the accelerate3DEnabled of "%s" to %s' % (self.sName, fEnabled)); + self.oTstDrv.processPendingEvents(); + return fRc; + + def setCpuCount(self, cCpus): + """ + Set the number of CPUs. + Returns True on success and False on failure. Error information is logged. + """ + fRc = True; + try: + self.o.machine.CPUCount = cCpus; + except: + reporter.errorXcpt('failed to set the CPU count of "%s" to %s' % (self.sName, cCpus)); + fRc = False; + else: + reporter.log('set the CPU count of "%s" to %s' % (self.sName, cCpus)); + self.oTstDrv.processPendingEvents(); + return fRc; + + def getCpuCount(self): + """ + Returns the number of CPUs. + Returns the number of CPUs on success and 0 on failure. Error information is logged. + """ + cCpus = 0; + try: + cCpus = self.o.machine.CPUCount; + except: + reporter.errorXcpt('failed to get the CPU count of "%s"' % (self.sName,)); + + self.oTstDrv.processPendingEvents(); + return cCpus; + + def ensureControllerAttached(self, sController): + """ + Makes sure the specified controller is attached to the VM, attaching it + if necessary. + """ + try: + try: + self.o.machine.getStorageControllerByName(sController); + except: + (eBus, eType) = _ControllerNameToBusAndType(sController); + try: + oCtl = self.o.machine.addStorageController(sController, eBus); + except: + reporter.errorXcpt('addStorageController("%s",%s) failed on "%s"' % (sController, eBus, self.sName) ); + return False; + else: + try: + oCtl.controllerType = eType; + reporter.log('added storage controller "%s" (bus %s, type %s) to %s' + % (sController, eBus, eType, self.sName)); + except: + reporter.errorXcpt('controllerType = %s on ("%s" / %s) failed on "%s"' + % (eType, sController, eBus, self.sName) ); + return False; + finally: + self.oTstDrv.processPendingEvents(); + return True; + + def setStorageControllerPortCount(self, sController, iPortCount): + """ + Set maximum ports count for storage controller + """ + try: + oCtl = self.o.machine.getStorageControllerByName(sController) + oCtl.portCount = iPortCount + self.oTstDrv.processPendingEvents() + reporter.log('set controller "%s" port count to value %d' % (sController, iPortCount)) + return True + except: + reporter.log('unable to set storage controller "%s" ports count to %d' % (sController, iPortCount)) + + return False + + def setStorageControllerHostIoCache(self, sController, fUseHostIoCache): + """ + Set maximum ports count for storage controller + """ + try: + oCtl = self.o.machine.getStorageControllerByName(sController); + oCtl.useHostIOCache = fUseHostIoCache; + self.oTstDrv.processPendingEvents(); + reporter.log('set controller "%s" host I/O cache setting to %r' % (sController, fUseHostIoCache)); + return True; + except: + reporter.log('unable to set storage controller "%s" host I/O cache setting to %r' % (sController, fUseHostIoCache)); + + return False; + + def setBootOrder(self, iPosition, eType): + """ + Set guest boot order type + @param iPosition boot order position + @param eType device type (vboxcon.DeviceType_HardDisk, + vboxcon.DeviceType_DVD, vboxcon.DeviceType_Floppy) + """ + try: + self.o.machine.setBootOrder(iPosition, eType) + except: + return reporter.errorXcpt('Unable to set boot order.') + + reporter.log('Set boot order [%d] for device %s' % (iPosition, str(eType))) + self.oTstDrv.processPendingEvents(); + + return True + + def setStorageControllerType(self, eType, sController = "IDE Controller"): + """ + Similar to ensureControllerAttached, except it will change the type. + """ + try: + oCtl = self.o.machine.getStorageControllerByName(sController); + except: + (eBus, _) = _ControllerNameToBusAndType(sController); + try: + oCtl = self.o.machine.addStorageController(sController, eBus); + reporter.log('added storage controller "%s" (bus %s) to %s' % (sController, eBus, self.sName)); + except: + reporter.errorXcpt('addStorageController("%s",%s) failed on "%s"' % (sController, eBus, self.sName) ); + return False; + try: + oCtl.controllerType = eType; + except: + reporter.errorXcpt('failed to set controller type of "%s" on "%s" to %s' % (sController, self.sName, eType) ); + return False; + reporter.log('set controller type of "%s" on "%s" to %s' % (sController, self.sName, eType) ); + self.oTstDrv.processPendingEvents(); + return True; + + def attachDvd(self, sImage = None, sController = "IDE Controller", iPort = 1, iDevice = 0): + """ + Attaches a DVD drive to a VM, optionally with an ISO inserted. + Returns True on success and False on failure. Error information is logged. + """ + # Input validation. + if sImage is not None and not self.oTstDrv.isResourceFile(sImage)\ + and not os.path.isabs(sImage): ## fixme - testsuite unzip ++ + reporter.fatal('"%s" is not in the resource set' % (sImage)); + return None; + + if not self.ensureControllerAttached(sController): + return False; + + # Find/register the image if specified. + oImage = None; + sImageUuid = ""; + if sImage is not None: + sFullName = self.oTstDrv.getFullResourceName(sImage) + try: + oImage = self.oVBox.findDVDImage(sFullName); + except: + try: + if self.fpApiVer >= 4.1: + oImage = self.oVBox.openMedium(sFullName, vboxcon.DeviceType_DVD, vboxcon.AccessMode_ReadOnly, False); + elif self.fpApiVer >= 4.0: + oImage = self.oVBox.openMedium(sFullName, vboxcon.DeviceType_DVD, vboxcon.AccessMode_ReadOnly); + else: + oImage = self.oVBox.openDVDImage(sFullName, ""); + except vbox.ComException as oXcpt: + if oXcpt.errno != -1: + reporter.errorXcpt('failed to open DVD image "%s" xxx' % (sFullName)); + else: + reporter.errorXcpt('failed to open DVD image "%s" yyy' % (sFullName)); + return False; + except: + reporter.errorXcpt('failed to open DVD image "%s"' % (sFullName)); + return False; + try: + sImageUuid = oImage.id; + except: + reporter.errorXcpt('failed to get the UUID of "%s"' % (sFullName)); + return False; + + # Attach the DVD. + fRc = True; + try: + if self.fpApiVer >= 4.0: + self.o.machine.attachDevice(sController, iPort, iDevice, vboxcon.DeviceType_DVD, oImage); + else: + self.o.machine.attachDevice(sController, iPort, iDevice, vboxcon.DeviceType_DVD, sImageUuid); + except: + reporter.errorXcpt('attachDevice("%s",%s,%s,HardDisk,"%s") failed on "%s"' \ + % (sController, iPort, iDevice, sImageUuid, self.sName) ); + fRc = False; + else: + reporter.log('attached DVD to %s, image="%s"' % (self.sName, sImage)); + self.oTstDrv.processPendingEvents(); + return fRc; + + def attachHd(self, sHd, sController = "IDE Controller", iPort = 0, iDevice = 0, fImmutable = True, fForceResource = True): + """ + Attaches a HD to a VM. + Returns True on success and False on failure. Error information is logged. + """ + # Input validation. + if fForceResource and not self.oTstDrv.isResourceFile(sHd): + reporter.fatal('"%s" is not in the resource set' % (sHd,)); + return None; + + if not self.ensureControllerAttached(sController): + return False; + + # Find the HD, registering it if necessary (as immutable). + if fForceResource: + sFullName = self.oTstDrv.getFullResourceName(sHd); + else: + sFullName = sHd; + try: + oHd = self.oVBox.findHardDisk(sFullName); + except: + try: + if self.fpApiVer >= 4.1: + oHd = self.oVBox.openMedium(sFullName, vboxcon.DeviceType_HardDisk, vboxcon.AccessMode_ReadOnly, False); + elif self.fpApiVer >= 4.0: + oHd = self.oVBox.openMedium(sFullName, vboxcon.DeviceType_HardDisk, vboxcon.AccessMode_ReadOnly); + else: + oHd = self.oVBox.openHardDisk(sFullName, vboxcon.AccessMode_ReadOnly, False, "", False, ""); + except: + reporter.errorXcpt('failed to open hd "%s"' % (sFullName)); + return False; + try: + if fImmutable: + oHd.type = vboxcon.MediumType_Immutable; + else: + oHd.type = vboxcon.MediumType_Normal; + except: + if fImmutable: + reporter.errorXcpt('failed to set hd "%s" immutable' % (sHd)); + else: + reporter.errorXcpt('failed to set hd "%s" normal' % (sHd)); + return False; + + # Attach it. + fRc = True; + try: + if self.fpApiVer >= 4.0: + self.o.machine.attachDevice(sController, iPort, iDevice, vboxcon.DeviceType_HardDisk, oHd); + else: + self.o.machine.attachDevice(sController, iPort, iDevice, vboxcon.DeviceType_HardDisk, oHd.id); + except: + reporter.errorXcpt('attachDevice("%s",%s,%s,HardDisk,"%s") failed on "%s"' \ + % (sController, iPort, iDevice, oHd.id, self.sName) ); + fRc = False; + else: + reporter.log('attached "%s" to %s' % (sHd, self.sName)); + self.oTstDrv.processPendingEvents(); + return fRc; + + def createBaseHd(self, sHd, sFmt = "VDI", cb = 10*1024*1024*1024, cMsTimeout = 60000, tMediumVariant = None): + """ + Creates a base HD. + Returns Medium object on success and None on failure. Error information is logged. + """ + if tMediumVariant is None: + tMediumVariant = (vboxcon.MediumVariant_Standard, ); + + try: + if self.fpApiVer >= 5.0: + oHd = self.oVBox.createMedium(sFmt, sHd, vboxcon.AccessMode_ReadWrite, vboxcon.DeviceType_HardDisk); + else: + oHd = self.oVBox.createHardDisk(sFmt, sHd); + oProgressXpcom = oHd.createBaseStorage(cb, tMediumVariant); + oProgress = ProgressWrapper(oProgressXpcom, self.oVBoxMgr, self.oTstDrv, 'create base disk %s' % (sHd)); + oProgress.wait(cMsTimeout); + oProgress.logResult(); + except: + reporter.errorXcpt('failed to create base hd "%s"' % (sHd)); + oHd = None + + return oHd; + + def createDiffHd(self, oParentHd, sHd, sFmt = "VDI"): + """ + Creates a differencing HD. + Returns Medium object on success and None on failure. Error information is logged. + """ + # Detect the proper format if requested + if sFmt is None: + try: + oHdFmt = oParentHd.mediumFormat; + lstCaps = self.oVBoxMgr.getArray(oHdFmt, 'capabilities'); + if vboxcon.MediumFormatCapabilities_Differencing in lstCaps: + sFmt = oHdFmt.id; + else: + sFmt = 'VDI'; + except: + reporter.errorXcpt('failed to get preferred diff format for "%s"' % (sHd)); + return None; + try: + if self.fpApiVer >= 5.0: + oHd = self.oVBox.createMedium(sFmt, sHd, vboxcon.AccessMode_ReadWrite, vboxcon.DeviceType_HardDisk); + else: + oHd = self.oVBox.createHardDisk(sFmt, sHd); + oProgressXpcom = oParentHd.createDiffStorage(oHd, (vboxcon.MediumVariant_Standard, )) + oProgress = ProgressWrapper(oProgressXpcom, self.oVBoxMgr, self.oTstDrv, 'create diff disk %s' % (sHd)); + oProgress.wait(); + oProgress.logResult(); + except: + reporter.errorXcpt('failed to create diff hd "%s"' % (sHd)); + oHd = None + + return oHd; + + def createAndAttachHd(self, sHd, sFmt = "VDI", sController = "IDE Controller", cb = 10*1024*1024*1024, # pylint: disable=too-many-arguments + iPort = 0, iDevice = 0, fImmutable = True, cMsTimeout = 60000, tMediumVariant = None): + """ + Creates and attaches a HD to a VM. + Returns True on success and False on failure. Error information is logged. + """ + if not self.ensureControllerAttached(sController): + return False; + + oHd = self.createBaseHd(sHd, sFmt, cb, cMsTimeout, tMediumVariant); + if oHd is None: + return False; + + fRc = True; + try: + if fImmutable: + oHd.type = vboxcon.MediumType_Immutable; + else: + oHd.type = vboxcon.MediumType_Normal; + except: + if fImmutable: + reporter.errorXcpt('failed to set hd "%s" immutable' % (sHd)); + else: + reporter.errorXcpt('failed to set hd "%s" normal' % (sHd)); + fRc = False; + + # Attach it. + if fRc is True: + try: + if self.fpApiVer >= 4.0: + self.o.machine.attachDevice(sController, iPort, iDevice, vboxcon.DeviceType_HardDisk, oHd); + else: + self.o.machine.attachDevice(sController, iPort, iDevice, vboxcon.DeviceType_HardDisk, oHd.id); + except: + reporter.errorXcpt('attachDevice("%s",%s,%s,HardDisk,"%s") failed on "%s"' \ + % (sController, iPort, iDevice, oHd.id, self.sName) ); + fRc = False; + else: + reporter.log('attached "%s" to %s' % (sHd, self.sName)); + + # Delete disk in case of an error + if fRc is False: + try: + oProgressCom = oHd.deleteStorage(); + except: + reporter.errorXcpt('deleteStorage() for disk %s failed' % (sHd,)); + else: + oProgress = ProgressWrapper(oProgressCom, self.oVBoxMgr, self.oTstDrv, 'delete disk %s' % (sHd)); + oProgress.wait(); + oProgress.logResult(); + + self.oTstDrv.processPendingEvents(); + return fRc; + + def detachHd(self, sController = "IDE Controller", iPort = 0, iDevice = 0): + """ + Detaches a HD, if attached, and returns a reference to it (IMedium). + + In order to delete the detached medium, the caller must first save + the changes made in this session. + + Returns (fRc, oHd), where oHd is None unless fRc is True, and fRc is + your standard success indicator. Error information is logged. + """ + + # What's attached? + try: + oHd = self.o.machine.getMedium(sController, iPort, iDevice); + except: + if self.oVBoxMgr.xcptIsOurXcptKind() \ + and self.oVBoxMgr.xcptIsEqual(None, self.oVBoxMgr.constants.VBOX_E_OBJECT_NOT_FOUND): + reporter.log('No HD attached (to %s %s:%s)' % (sController, iPort, iDevice)); + return (True, None); + return (reporter.errorXcpt('Error getting media at port %s, device %s, on %s.' + % (iPort, iDevice, sController)), None); + # Detach it. + try: + self.o.machine.detachDevice(sController, iPort, iDevice); + except: + return (reporter.errorXcpt('detachDevice("%s",%s,%s) failed on "%s"' \ + % (sController, iPort, iDevice, self.sName) ), None); + reporter.log('detached HD ("%s",%s,%s) from %s' % (sController, iPort, iDevice, self.sName)); + return (True, oHd); + + def attachFloppy(self, sFloppy, sController = "Floppy Controller", iPort = 0, iDevice = 0): + """ + Attaches a floppy image to a VM. + Returns True on success and False on failure. Error information is logged. + """ + # Input validation. + ## @todo Fix this wrt to bootsector-xxx.img from the validationkit.zip. + ##if not self.oTstDrv.isResourceFile(sFloppy): + ## reporter.fatal('"%s" is not in the resource set' % (sFloppy)); + ## return None; + + if not self.ensureControllerAttached(sController): + return False; + + # Find the floppy image, registering it if necessary (as immutable). + sFullName = self.oTstDrv.getFullResourceName(sFloppy); + try: + oFloppy = self.oVBox.findFloppyImage(sFullName); + except: + try: + if self.fpApiVer >= 4.1: + oFloppy = self.oVBox.openMedium(sFullName, vboxcon.DeviceType_Floppy, vboxcon.AccessMode_ReadOnly, False); + elif self.fpApiVer >= 4.0: + oFloppy = self.oVBox.openMedium(sFullName, vboxcon.DeviceType_Floppy, vboxcon.AccessMode_ReadOnly); + else: + oFloppy = self.oVBox.openFloppyImage(sFullName, ""); + except: + reporter.errorXcpt('failed to open floppy "%s"' % (sFullName)); + return False; + ## @todo the following works but causes trouble below (asserts in main). + #try: + # oFloppy.type = vboxcon.MediumType_Immutable; + #except: + # reporter.errorXcpt('failed to make floppy "%s" immutable' % (sFullName)); + # return False; + + # Attach it. + fRc = True; + try: + if self.fpApiVer >= 4.0: + self.o.machine.attachDevice(sController, iPort, iDevice, vboxcon.DeviceType_Floppy, oFloppy); + else: + self.o.machine.attachDevice(sController, iPort, iDevice, vboxcon.DeviceType_Floppy, oFloppy.id); + except: + reporter.errorXcpt('attachDevice("%s",%s,%s,Floppy,"%s") failed on "%s"' \ + % (sController, iPort, iDevice, oFloppy.id, self.sName) ); + fRc = False; + else: + reporter.log('attached "%s" to %s' % (sFloppy, self.sName)); + self.oTstDrv.processPendingEvents(); + return fRc; + + def setupNic(self, sType, sXXX): + """ + Sets up a NIC to a VM. + Returns True on success and False on failure. Error information is logged. + """ + if sType == "PCNet": enmType = vboxcon.NetworkAdapterType_Am79C973; + elif sType == "PCNetOld": enmType = vboxcon.NetworkAdapterType_Am79C970A; + elif sType == "E1000": enmType = vboxcon.NetworkAdapterType_I82545EM; # MT Server + elif sType == "E1000Desk": enmType = vboxcon.NetworkAdapterType_I82540EM; # MT Desktop + elif sType == "E1000Srv2": enmType = vboxcon.NetworkAdapterType_I82543GC; # T Server + elif sType == "Virtio": enmType = vboxcon.NetworkAdapterType_Virtio; + else: + reporter.error('Invalid NIC type: "%s" (sXXX=%s)' % (sType, sXXX)); + return False; + ## @todo Implement me! + if enmType is not None: pass + return True; + + def setupAudio(self, eAudioControllerType, fEnable = True, fEnableIn = False, fEnableOut = True, eAudioDriverType = None): + """ + Sets up audio. + + :param eAudioControllerType: The audio controller type (vboxcon.AudioControllerType_XXX). + :param fEnable: Whether to enable or disable the audio controller (default enable). + :param fEnableIn: Whether to enable or disable audio input (default disable). + :param fEnableOut: Whether to enable or disable audio output (default enable). + :param eAudioDriverType: The audio driver type (vboxcon.AudioDriverType_XXX), picks something suitable + if None is passed (default). + """ + try: + if self.fpApiVer >= 7.0: + oAdapter = self.o.machine.audioSettings.adapter; + else: + oAdapter = self.o.machine.audioAdapter; + except: return reporter.errorXcpt('Failed to get the audio adapter.'); + + try: oAdapter.audioController = eAudioControllerType; + except: return reporter.errorXcpt('Failed to set the audio controller to %s.' % (eAudioControllerType,)); + + if eAudioDriverType is None: + sHost = utils.getHostOs() + if sHost == 'darwin': eAudioDriverType = vboxcon.AudioDriverType_CoreAudio; + elif sHost == 'win': eAudioDriverType = vboxcon.AudioDriverType_DirectSound; + elif sHost == 'linux': eAudioDriverType = vboxcon.AudioDriverType_Pulse; + elif sHost == 'solaris': eAudioDriverType = vboxcon.AudioDriverType_OSS; + else: + reporter.error('PORTME: Do not know which audio driver to pick for: %s!' % (sHost,)); + eAudioDriverType = vboxcon.AudioDriverType_Null; + + try: oAdapter.audioDriver = eAudioDriverType; + except: return reporter.errorXcpt('Failed to set the audio driver to %s.' % (eAudioDriverType,)) + + try: oAdapter.enabled = fEnable; + except: return reporter.errorXcpt('Failed to set the "enabled" property to %s.' % (fEnable,)); + + try: oAdapter.enabledIn = fEnableIn; + except: return reporter.errorXcpt('Failed to set the "enabledIn" property to %s.' % (fEnable,)); + + try: oAdapter.enabledOut = fEnableOut; + except: return reporter.errorXcpt('Failed to set the "enabledOut" property to %s.' % (fEnable,)); + + reporter.log('set audio adapter type to %d, driver to %d, and enabled to %s (input is %s, output is %s)' + % (eAudioControllerType, eAudioDriverType, fEnable, fEnableIn, fEnableOut,)); + self.oTstDrv.processPendingEvents(); + return True; + + def setupPreferredConfig(self): # pylint: disable=too-many-locals + """ + Configures the VM according to the preferences of the guest type. + """ + try: + sOsTypeId = self.o.machine.OSTypeId; + except: + reporter.errorXcpt('failed to obtain the OSTypeId for "%s"' % (self.sName)); + return False; + + try: + oOsType = self.oVBox.getGuestOSType(sOsTypeId); + except: + reporter.errorXcpt('getGuestOSType("%s") failed for "%s"' % (sOsTypeId, self.sName)); + return False; + + # get the attributes. + try: + #sFamilyId = oOsType.familyId; + #f64Bit = oOsType.is64Bit; + fIoApic = oOsType.recommendedIOAPIC; + fVirtEx = oOsType.recommendedVirtEx; + cMBRam = oOsType.recommendedRAM; + cMBVRam = oOsType.recommendedVRAM; + #cMBHdd = oOsType.recommendedHDD; + eNicType = oOsType.adapterType; + if self.fpApiVer >= 3.2: + if self.fpApiVer >= 4.2: + fPae = oOsType.recommendedPAE; + fUsbHid = oOsType.recommendedUSBHID; + fHpet = oOsType.recommendedHPET; + eStorCtlType = oOsType.recommendedHDStorageController; + else: + fPae = oOsType.recommendedPae; + fUsbHid = oOsType.recommendedUsbHid; + fHpet = oOsType.recommendedHpet; + eStorCtlType = oOsType.recommendedHdStorageController; + eFirmwareType = oOsType.recommendedFirmware; + else: + fPae = False; + fUsbHid = False; + fHpet = False; + eFirmwareType = -1; + eStorCtlType = vboxcon.StorageControllerType_PIIX4; + if self.fpApiVer >= 4.0: + eAudioCtlType = oOsType.recommendedAudioController; + except: + reporter.errorXcpt('exception reading IGuestOSType(%s) attribute' % (sOsTypeId)); + self.oTstDrv.processPendingEvents(); + return False; + self.oTstDrv.processPendingEvents(); + + # Do the setting. Continue applying settings on error in case the + # caller ignores the return code + fRc = True; + if not self.enableIoApic(fIoApic): fRc = False; + if not self.enableVirtEx(fVirtEx): fRc = False; + if not self.enablePae(fPae): fRc = False; + if not self.setRamSize(cMBRam): fRc = False; + if not self.setVRamSize(cMBVRam): fRc = False; + if not self.setNicType(eNicType, 0): fRc = False; + if self.fpApiVer >= 3.2: + if not self.setFirmwareType(eFirmwareType): fRc = False; + if not self.enableUsbHid(fUsbHid): fRc = False; + if not self.enableHpet(fHpet): fRc = False; + if eStorCtlType in (vboxcon.StorageControllerType_PIIX3, + vboxcon.StorageControllerType_PIIX4, + vboxcon.StorageControllerType_ICH6,): + if not self.setStorageControllerType(eStorCtlType, "IDE Controller"): + fRc = False; + if self.fpApiVer >= 4.0: + if not self.setupAudio(eAudioCtlType): fRc = False; + + return fRc; + + def addUsbDeviceFilter(self, sName, sVendorId = None, sProductId = None, sRevision = None, # pylint: disable=too-many-arguments + sManufacturer = None, sProduct = None, sSerialNumber = None, + sPort = None, sRemote = None): + """ + Creates a USB device filter and inserts it into the VM. + Returns True on success. + Returns False on failure (logged). + """ + fRc = True; + + try: + oUsbDevFilter = self.o.machine.USBDeviceFilters.createDeviceFilter(sName); + oUsbDevFilter.active = True; + if sVendorId is not None: + oUsbDevFilter.vendorId = sVendorId; + if sProductId is not None: + oUsbDevFilter.productId = sProductId; + if sRevision is not None: + oUsbDevFilter.revision = sRevision; + if sManufacturer is not None: + oUsbDevFilter.manufacturer = sManufacturer; + if sProduct is not None: + oUsbDevFilter.product = sProduct; + if sSerialNumber is not None: + oUsbDevFilter.serialnumber = sSerialNumber; + if sPort is not None: + oUsbDevFilter.port = sPort; + if sRemote is not None: + oUsbDevFilter.remote = sRemote; + try: + self.o.machine.USBDeviceFilters.insertDeviceFilter(0, oUsbDevFilter); + except: + reporter.errorXcpt('insertDeviceFilter(%s) failed on "%s"' \ + % (0, self.sName) ); + fRc = False; + else: + reporter.log('inserted USB device filter "%s" to %s' % (sName, self.sName)); + except: + reporter.errorXcpt('createDeviceFilter("%s") failed on "%s"' \ + % (sName, self.sName) ); + fRc = False; + return fRc; + + def getGuestPropertyValue(self, sName): + """ + Gets a guest property value. + Returns the value on success, None on failure (logged). + """ + try: + sValue = self.o.machine.getGuestPropertyValue(sName); + except: + reporter.errorXcpt('IMachine::getGuestPropertyValue("%s") failed' % (sName)); + return None; + return sValue; + + def setGuestPropertyValue(self, sName, sValue): + """ + Sets a guest property value. + Returns the True on success, False on failure (logged). + """ + try: + self.o.machine.setGuestPropertyValue(sName, sValue); + except: + reporter.errorXcpt('IMachine::setGuestPropertyValue("%s","%s") failed' % (sName, sValue)); + return False; + return True; + + def delGuestPropertyValue(self, sName): + """ + Deletes a guest property value. + Returns the True on success, False on failure (logged). + """ + try: + oMachine = self.o.machine; + if self.fpApiVer >= 4.2: + oMachine.deleteGuestProperty(sName); + else: + oMachine.setGuestPropertyValue(sName, ''); + except: + reporter.errorXcpt('Unable to delete guest property "%s"' % (sName,)); + return False; + return True; + + def setExtraData(self, sKey, sValue): + """ + Sets extra data. + Returns the True on success, False on failure (logged). + """ + try: + self.o.machine.setExtraData(sKey, sValue); + except: + reporter.errorXcpt('IMachine::setExtraData("%s","%s") failed' % (sKey, sValue)); + return False; + return True; + + def getExtraData(self, sKey): + """ + Gets extra data. + Returns value on success, None on failure. + """ + try: + sValue = self.o.machine.getExtraData(sKey) + except: + reporter.errorXcpt('IMachine::setExtraData("%s","%s") failed' % (sKey, sValue)) + return None + return sValue + + def setupTeleporter(self, fEnabled=True, uPort = 6500, sAddress = '', sPassword = ''): + """ + Sets up the teleporter for the VM. + Returns True on success, False on failure (logged). + """ + try: + self.o.machine.teleporterAddress = sAddress; + self.o.machine.teleporterPort = uPort; + self.o.machine.teleporterPassword = sPassword; + self.o.machine.teleporterEnabled = fEnabled; + except: + reporter.errorXcpt('setupTeleporter(%s, %s, %s, %s)' % (fEnabled, sPassword, uPort, sAddress)); + return False; + return True; + + def enableTeleporter(self, fEnable=True): + """ + Enables or disables the teleporter of the VM. + Returns True on success, False on failure (logged). + """ + try: + self.o.machine.teleporterEnabled = fEnable; + except: + reporter.errorXcpt('IMachine::teleporterEnabled=%s failed' % (fEnable)); + return False; + return True; + + def teleport(self, sHostname = 'localhost', uPort = 6500, sPassword = 'password', cMsMaxDowntime = 250): + """ + Wrapper around the IConsole::teleport() method. + Returns a progress object on success, None on failure (logged). + """ + reporter.log2('"%s"::teleport(%s,%s,%s,%s)...' % (self.sName, sHostname, uPort, sPassword, cMsMaxDowntime)); + try: + oProgress = self.o.console.teleport(sHostname, uPort, sPassword, cMsMaxDowntime) + except: + reporter.errorXcpt('IConsole::teleport(%s,%s,%s,%s) failed' % (sHostname, uPort, sPassword, cMsMaxDowntime)); + return None; + return ProgressWrapper(oProgress, self.oVBoxMgr, self.oTstDrv, 'teleport %s' % (self.sName,)); + + def getOsType(self): + """ + Gets the IGuestOSType interface for the machine. + + return IGuestOSType interface on success, None + errorXcpt on failure. + No exceptions raised. + """ + try: + sOsTypeId = self.o.machine.OSTypeId; + except: + reporter.errorXcpt('failed to obtain the OSTypeId for "%s"' % (self.sName)); + return None; + + try: + oOsType = self.oVBox.getGuestOSType(sOsTypeId); + except: + reporter.errorXcpt('getGuestOSType("%s") failed for "%s"' % (sOsTypeId, self.sName)); + return None; + + return oOsType; + + def setOsType(self, sNewTypeId): + """ + Changes the OS type. + + returns True on success, False + errorXcpt on failure. + No exceptions raised. + """ + try: + self.o.machine.OSTypeId = sNewTypeId; + except: + reporter.errorXcpt('failed to set the OSTypeId for "%s" to "%s"' % (self.sName, sNewTypeId)); + return False; + return True; + + + def setParavirtProvider(self, iProvider): + """ + Sets a paravirtualisation provider. + Returns the True on success, False on failure (logged). + """ + try: + self.o.machine.paravirtProvider = iProvider + except: + reporter.errorXcpt('Unable to set paravirtualisation provider "%s"' % (iProvider,)) + return False; + return True; + + + def setupSerialToRawFile(self, iSerialPort, sRawFile): + """ + Enables the given serial port (zero based) and redirects it to sRawFile. + Returns the True on success, False on failure (logged). + """ + try: + oPort = self.o.machine.getSerialPort(iSerialPort); + except: + fRc = reporter.errorXcpt('failed to get serial port #%u' % (iSerialPort,)); + else: + try: + oPort.path = sRawFile; + except: + fRc = reporter.errorXcpt('failed to set the "path" property on serial port #%u to "%s"' + % (iSerialPort, sRawFile)); + else: + try: + oPort.hostMode = vboxcon.PortMode_RawFile; + except: + fRc = reporter.errorXcpt('failed to set the "hostMode" property on serial port #%u to PortMode_RawFile' + % (iSerialPort,)); + else: + try: + oPort.enabled = True; + except: + fRc = reporter.errorXcpt('failed to set the "enable" property on serial port #%u to True' + % (iSerialPort,)); + else: + reporter.log('set SerialPort[%s].enabled/hostMode/path=True/RawFile/%s' % (iSerialPort, sRawFile,)); + fRc = True; + self.oTstDrv.processPendingEvents(); + return fRc; + + + def enableSerialPort(self, iSerialPort): + """ + Enables the given serial port setting the initial port mode to disconnected. + """ + try: + oPort = self.o.machine.getSerialPort(iSerialPort); + except: + fRc = reporter.errorXcpt('failed to get serial port #%u' % (iSerialPort,)); + else: + try: + oPort.hostMode = vboxcon.PortMode_Disconnected; + except: + fRc = reporter.errorXcpt('failed to set the "hostMode" property on serial port #%u to PortMode_Disconnected' + % (iSerialPort,)); + else: + try: + oPort.enabled = True; + except: + fRc = reporter.errorXcpt('failed to set the "enable" property on serial port #%u to True' + % (iSerialPort,)); + else: + reporter.log('set SerialPort[%s].enabled/hostMode/=True/Disconnected' % (iSerialPort,)); + fRc = True; + self.oTstDrv.processPendingEvents(); + return fRc; + + + def changeSerialPortAttachment(self, iSerialPort, ePortMode, sPath, fServer): + """ + Changes the attachment of the given serial port to the attachment config given. + """ + try: + oPort = self.o.machine.getSerialPort(iSerialPort); + except: + fRc = reporter.errorXcpt('failed to get serial port #%u' % (iSerialPort,)); + else: + try: + # Change port mode to disconnected first so changes get picked up by a potentially running VM. + oPort.hostMode = vboxcon.PortMode_Disconnected; + except: + fRc = reporter.errorXcpt('failed to set the "hostMode" property on serial port #%u to PortMode_Disconnected' + % (iSerialPort,)); + else: + try: + oPort.path = sPath; + oPort.server = fServer; + oPort.hostMode = ePortMode; + except: + fRc = reporter.errorXcpt('failed to configure the serial port'); + else: + reporter.log('set SerialPort[%s].hostMode/path/server=%s/%s/%s' + % (iSerialPort, ePortMode, sPath, fServer)); + fRc = True; + self.oTstDrv.processPendingEvents(); + return fRc; + + # + # IConsole wrappers. + # + + def powerOff(self, fFudgeOnFailure = True): + """ + Powers off the VM. + + Returns True on success. + Returns False on IConsole::powerDown() failure. + Returns None if the progress object returns failure. + """ + # + # Deregister event handler before we power off the VM, otherwise we're + # racing for VM process termination and cause misleading spurious + # error messages in the event handling code, because the event objects + # disappear. + # + # Note! Doing this before powerDown to try prevent numerous smoketest + # timeouts on XPCOM hosts. + # + self.deregisterEventHandlerForTask(); + + + # Try power if off. + try: + oProgress = self.o.console.powerDown(); + except: + reporter.logXcpt('IConsole::powerDown failed on %s' % (self.sName)); + if fFudgeOnFailure: + self.oTstDrv.waitOnDirectSessionClose(self.oVM, 5000); # fudge + self.waitForTask(1000); # fudge + return False; + + # Wait on power off operation to complete. + rc = self.oTstDrv.waitOnProgress(oProgress); + if rc < 0: + self.close(); + if fFudgeOnFailure: + vbox.reportError(oProgress, 'powerDown for "%s" failed' % (self.sName)); + self.oTstDrv.waitOnDirectSessionClose(self.oVM, 5000); # fudge + return None; + + # Wait for the VM to really power off or we'll fail to open a new session to it. + self.oTstDrv.waitOnDirectSessionClose(self.oVM, 5000); # fudge + return self.waitForTask(30 * 1000); # fudge + + def saveState(self, fPause = True): + """ + Saves state of the VM. + + Returns True on success. + Returns False on IConsole::saveState() failure. + Returns None if the progress object returns Failure. + """ + + if fPause is True \ + and self.oVM.state is vboxcon.MachineState_Running: + self.o.console.pause(); + if self.oVM.state is not vboxcon.MachineState_Paused: + reporter.error('pause for "%s" failed' % (self.sName)); + # Try saving state. + try: + if self.fpApiVer >= 5.0: + oProgress = self.o.machine.saveState() + else: + oProgress = self.o.console.saveState() + except: + reporter.logXcpt('IMachine::saveState failed on %s' % (self.sName)); + return False; + + # Wait for saving state operation to complete. + rc = self.oTstDrv.waitOnProgress(oProgress); + if rc < 0: + self.close(); + return None; + + # Wait for the VM to really terminate or we'll fail to open a new session to it. + self.oTstDrv.waitOnDirectSessionClose(self.oVM, 5000); # fudge + return self.waitForTask(30 * 1000); # fudge + + def discardSavedState(self, fRemove = True): + """ + Discards saved state of the VM. + + Returns True on success. + Returns False on IConsole::discardSaveState() failure. + """ + + try: + if self.fpApiVer >= 5.0: + self.o.machine.discardSavedState(fRemove) + else: + self.o.console.discardSavedState(fRemove) + except: + reporter.logXcpt('IMachine::discardSavedState failed on %s' % (self.sName)) + return False + return True + + def restoreSnapshot(self, oSnapshot, fFudgeOnFailure = True): + """ + Restores the given snapshot. + + Returns True on success. + Returns False on IMachine::restoreSnapshot() failure. + Returns None if the progress object returns failure. + """ + try: + if self.fpApiVer >= 5.0: + oProgress = self.o.machine.restoreSnapshot(oSnapshot); + else: + oProgress = self.o.console.restoreSnapshot(oSnapshot); + except: + reporter.logXcpt('IMachine::restoreSnapshot failed on %s' % (self.sName)); + if fFudgeOnFailure: + self.oTstDrv.waitOnDirectSessionClose(self.oVM, 5000); # fudge + self.waitForTask(1000); # fudge + return False; + + rc = self.oTstDrv.waitOnProgress(oProgress); + if rc < 0: + self.close(); + if fFudgeOnFailure: + vbox.reportError(oProgress, 'restoreSnapshot for "%s" failed' % (self.sName)); + return None; + + return self.waitForTask(30 * 1000); + + def deleteSnapshot(self, oSnapshot, fFudgeOnFailure = True, cMsTimeout = 30 * 1000): + """ + Deletes the given snapshot merging the diff image into the base. + + Returns True on success. + Returns False on IMachine::deleteSnapshot() failure. + """ + try: + if self.fpApiVer >= 5.0: + oProgressCom = self.o.machine.deleteSnapshot(oSnapshot); + else: + oProgressCom = self.o.console.deleteSnapshot(oSnapshot); + oProgress = ProgressWrapper(oProgressCom, self.oVBoxMgr, self.oTstDrv, 'Delete Snapshot %s' % (oSnapshot)); + oProgress.wait(cMsTimeout); + oProgress.logResult(); + except: + reporter.logXcpt('IMachine::deleteSnapshot failed on %s' % (self.sName)); + if fFudgeOnFailure: + self.oTstDrv.waitOnDirectSessionClose(self.oVM, 5000); # fudge + self.waitForTask(1000); # fudge + return False; + + return True; + + def takeSnapshot(self, sName, sDescription = '', fPause = True, fFudgeOnFailure = True, cMsTimeout = 30 * 1000): + """ + Takes a snapshot with the given name + + Returns True on success. + Returns False on IMachine::takeSnapshot() or VM state change failure. + """ + try: + if fPause is True \ + and self.oVM.state is vboxcon.MachineState_Running: + self.o.console.pause(); + if self.fpApiVer >= 5.0: + (oProgressCom, _) = self.o.machine.takeSnapshot(sName, sDescription, True); + else: + oProgressCom = self.o.console.takeSnapshot(sName, sDescription); + oProgress = ProgressWrapper(oProgressCom, self.oVBoxMgr, self.oTstDrv, 'Take Snapshot %s' % (sName)); + oProgress.wait(cMsTimeout); + oProgress.logResult(); + except: + reporter.logXcpt('IMachine::takeSnapshot failed on %s' % (self.sName)); + if fFudgeOnFailure: + self.oTstDrv.waitOnDirectSessionClose(self.oVM, 5000); # fudge + self.waitForTask(1000); # fudge + return False; + + if fPause is True \ + and self.oVM.state is vboxcon.MachineState_Paused: + self.o.console.resume(); + + return True; + + def findSnapshot(self, sName): + """ + Returns the snapshot object with the given name + + Returns snapshot object on success. + Returns None if there is no snapshot with the given name. + """ + return self.oVM.findSnapshot(sName); + + def takeScreenshot(self, sFilename, iScreenId=0): + """ + Take screenshot from the given display and save it to specified file. + + Returns True on success + Returns False on failure. + """ + try: + if self.fpApiVer >= 5.0: + iWidth, iHeight, _, _, _, _ = self.o.console.display.getScreenResolution(iScreenId) + aPngData = self.o.console.display.takeScreenShotToArray(iScreenId, iWidth, iHeight, + vboxcon.BitmapFormat_PNG) + else: + iWidth, iHeight, _, _, _ = self.o.console.display.getScreenResolution(iScreenId) + aPngData = self.o.console.display.takeScreenShotPNGToArray(iScreenId, iWidth, iHeight) + except: + reporter.logXcpt("Unable to take screenshot") + return False + + with open(sFilename, 'wb') as oFile: # pylint: disable=unspecified-encoding + oFile.write(aPngData) + + return True + + def attachUsbDevice(self, sUuid, sCaptureFilename = None): + """ + Attach given USB device UUID to the VM. + + Returns True on success + Returns False on failure. + """ + fRc = True; + try: + if sCaptureFilename is None: + self.o.console.attachUSBDevice(sUuid, ''); + else: + self.o.console.attachUSBDevice(sUuid, sCaptureFilename); + except: + reporter.logXcpt('Unable to attach USB device %s' % (sUuid,)); + fRc = False; + + return fRc; + + def detachUsbDevice(self, sUuid): + """ + Detach given USB device UUID from the VM. + + Returns True on success + Returns False on failure. + """ + fRc = True; + try: + _ = self.o.console.detachUSBDevice(sUuid); + except: + reporter.logXcpt('Unable to detach USB device %s' % (sUuid,)); + fRc = False; + + return fRc; + + + # + # IMachineDebugger wrappers. + # + + def queryOsKernelLog(self): + """ + Tries to get the OS kernel log using the VM debugger interface. + + Returns string containing the kernel log on success. + Returns None on failure. + """ + sOsKernelLog = None; + try: + self.o.console.debugger.loadPlugIn('all'); + except: + reporter.logXcpt('Unable to load debugger plugins'); + else: + try: + sOsDetected = self.o.console.debugger.detectOS(); + except: + reporter.logXcpt('Failed to detect the guest OS'); + else: + try: + sOsKernelLog = self.o.console.debugger.queryOSKernelLog(0); + except: + reporter.logXcpt('Unable to get the guest OS (%s) kernel log' % (sOsDetected,)); + return sOsKernelLog; + + def queryDbgInfo(self, sItem, sArg = '', sDefault = None): + """ + Simple wrapper around IMachineDebugger::info. + + Returns string on success, sDefault on failure (logged). + """ + try: + return self.o.console.debugger.info(sItem, sArg); + except: + reporter.logXcpt('Unable to query "%s" with arg "%s"' % (sItem, sArg,)); + return sDefault; + + def queryDbgInfoVgaText(self, sArg = 'all'): + """ + Tries to get the 'info vgatext' output, provided we're in next mode. + + Returns string containing text on success. + Returns None on failure or not text mode. + """ + sVgaText = None; + try: + sVgaText = self.o.console.debugger.info('vgatext', sArg); + if sVgaText.startswith('Not in text mode!'): + sVgaText = None; + except: + reporter.logXcpt('Unable to query vgatext with arg "%s"' % (sArg,)); + return sVgaText; + + def queryDbgGuestStack(self, iCpu = 0): + """ + Returns the guest stack for the given VCPU. + + Returns string containing the guest stack for the selected VCPU on success. + Returns None on failure. + """ + + # + # Load all plugins first and try to detect the OS so we can + # get nicer stack traces. + # + try: + self.o.console.debugger.loadPlugIn('all'); + except: + reporter.logXcpt('Unable to load debugger plugins'); + else: + try: + sOsDetected = self.o.console.debugger.detectOS(); + _ = sOsDetected; + except: + reporter.logXcpt('Failed to detect the guest OS'); + + sGuestStack = None; + try: + sGuestStack = self.o.console.debugger.dumpGuestStack(iCpu); + except: + reporter.logXcpt('Unable to query guest stack for CPU %s' % (iCpu, )); + + return sGuestStack; + + + # + # Other methods. + # + + def getPrimaryIp(self): + """ + Tries to obtain the primary IP address of the guest via the guest + properties. + + Returns IP address on success. + Returns empty string on failure. + """ + sIpAddr = self.getGuestPropertyValue('/VirtualBox/GuestInfo/Net/0/V4/IP'); + if vbox.isIpAddrValid(sIpAddr): + return sIpAddr; + return ''; + + def getPid(self): + """ + Gets the process ID for the direct session unless it's ourselves. + """ + if self.uPid is None and self.o is not None and self.fRemoteSession: + try: + if self.fpApiVer >= 4.2: + uPid = self.o.machine.sessionPID; + else: + uPid = self.o.machine.sessionPid; + if uPid != os.getpid() and uPid != 0xffffffff: + self.uPid = uPid; + except Exception as oXcpt: + if vbox.ComError.equal(oXcpt, vbox.ComError.E_UNEXPECTED): + try: + if self.fpApiVer >= 4.2: + uPid = self.oVM.sessionPID; + else: + uPid = self.oVM.sessionPid; + if uPid != os.getpid() and uPid != 0xffffffff: + self.uPid = uPid; + except: + reporter.log2Xcpt(); + else: + reporter.log2Xcpt(); + if self.uPid is not None: + reporter.log2('getPid: %u' % (self.uPid,)); + self.fPidFile = self.oTstDrv.pidFileAdd(self.uPid, 'vm_%s' % (self.sName,), # Set-uid-to-root is similar to SUDO. + fSudo = True); + return self.uPid; + + def addLogsToReport(self, cReleaseLogs = 1): + """ + Retrieves and adds the release and debug logs to the test report. + """ + fRc = True; + + # Add each of the requested release logs to the report. + for iLog in range(0, cReleaseLogs): + try: + if self.fpApiVer >= 3.2: + sLogFile = self.oVM.queryLogFilename(iLog); + elif iLog > 0: + sLogFile = '%s/VBox.log' % (self.oVM.logFolder,); + else: + sLogFile = '%s/VBox.log.%u' % (self.oVM.logFolder, iLog); + except: + reporter.logXcpt('iLog=%s' % (iLog,)); + fRc = False; + else: + if sLogFile is not None and sLogFile != '': # the None bit is for a 3.2.0 bug. + reporter.addLogFile(sLogFile, 'log/release/vm', '%s #%u' % (self.sName, iLog), + sAltName = '%s-%s' % (self.sName, os.path.basename(sLogFile),)); + + # Now for the hardened windows startup log. + try: + sLogFile = os.path.join(self.oVM.logFolder, 'VBoxHardening.log'); + except: + reporter.logXcpt(); + fRc = False; + else: + if os.path.isfile(sLogFile): + reporter.addLogFile(sLogFile, 'log/release/vm', '%s hardening log' % (self.sName, ), + sAltName = '%s-%s' % (self.sName, os.path.basename(sLogFile),)); + + # Now for the debug log. + if self.sLogFile is not None and os.path.isfile(self.sLogFile): + reporter.addLogFile(self.sLogFile, 'log/debug/vm', '%s debug' % (self.sName, ), + sAltName = '%s-%s' % (self.sName, os.path.basename(self.sLogFile),)); + + return fRc; + + def registerDerivedEventHandler(self, oSubClass, dArgs = None, fMustSucceed = True): + """ + Create an instance of the given ConsoleEventHandlerBase sub-class and + register it. + + The new instance is returned on success. None is returned on error. + """ + + # We need a console object. + try: + oConsole = self.o.console; + except Exception as oXcpt: + if fMustSucceed or vbox.ComError.notEqual(oXcpt, vbox.ComError.E_UNEXPECTED): + reporter.errorXcpt('Failed to get ISession::console for "%s"' % (self.sName, )); + return None; + + # Add the base class arguments. + dArgsCopy = dArgs.copy() if dArgs is not None else {}; + dArgsCopy['oSession'] = self; + dArgsCopy['oConsole'] = oConsole; + sLogSuffix = 'on %s' % (self.sName,) + return oSubClass.registerDerivedEventHandler(self.oVBoxMgr, self.fpApiVer, oSubClass, dArgsCopy, + oConsole, 'IConsole', 'IConsoleCallback', + fMustSucceed = fMustSucceed, sLogSuffix = sLogSuffix); + + def enableVmmDevTestingPart(self, fEnabled, fEnableMMIO = False): + """ + Enables the testing part of the VMMDev. + + Returns True on success and False on failure. Error information is logged. + """ + fRc = True; + try: + self.o.machine.setExtraData('VBoxInternal/Devices/VMMDev/0/Config/TestingEnabled', + '1' if fEnabled else ''); + self.o.machine.setExtraData('VBoxInternal/Devices/VMMDev/0/Config/TestingMMIO', + '1' if fEnableMMIO and fEnabled else ''); + except: + reporter.errorXcpt('VM name "%s", fEnabled=%s' % (self.sName, fEnabled)); + fRc = False; + else: + reporter.log('set VMMDevTesting=%s for "%s"' % (fEnabled, self.sName)); + self.oTstDrv.processPendingEvents(); + return fRc; + + # + # Test eXecution Service methods. + # + + def txsConnectViaTcp(self, cMsTimeout = 10*60000, sIpAddr = None, fNatForwardingForTxs = False): + """ + Connects to the TXS using TCP/IP as transport. If no IP or MAC is + addresses are specified, we'll get the IP from the guest additions. + + Returns a TxsConnectTask object on success, None + log on failure. + """ + # If the VM is configured with a NAT interface, connect to local host. + fReversedSetup = False; + fUseNatForTxs = False; + sMacAddr = None; + oIDhcpServer = None; + if sIpAddr is None: + try: + oNic = self.oVM.getNetworkAdapter(0); + enmAttachmentType = oNic.attachmentType; + if enmAttachmentType == vboxcon.NetworkAttachmentType_NAT: + fUseNatForTxs = True; + elif enmAttachmentType == vboxcon.NetworkAttachmentType_HostOnly and not sIpAddr: + # Get the MAC address and find the DHCP server. + sMacAddr = oNic.MACAddress; + sHostOnlyNIC = oNic.hostOnlyInterface; + oIHostOnlyIf = self.oVBox.host.findHostNetworkInterfaceByName(sHostOnlyNIC); + sHostOnlyNet = oIHostOnlyIf.networkName; + oIDhcpServer = self.oVBox.findDHCPServerByNetworkName(sHostOnlyNet); + except: + reporter.errorXcpt(); + return None; + + if fUseNatForTxs: + fReversedSetup = not fNatForwardingForTxs; + sIpAddr = '127.0.0.1'; + + # Kick off the task. + try: + oTask = TxsConnectTask(self, cMsTimeout, sIpAddr, sMacAddr, oIDhcpServer, fReversedSetup, + fnProcessEvents = self.oTstDrv.processPendingEvents); + except: + reporter.errorXcpt(); + oTask = None; + return oTask; + + def txsTryConnectViaTcp(self, cMsTimeout, sHostname, fReversed = False): + """ + Attempts to connect to a TXS instance. + + Returns True if a connection was established, False if not (only grave + failures are logged as errors). + + Note! The timeout is more of a guideline... + """ + + if sHostname is None or sHostname.strip() == '': + raise base.GenError('Empty sHostname is not implemented yet'); + + oTxsSession = txsclient.tryOpenTcpSession(cMsTimeout, sHostname, fReversedSetup = fReversed, + cMsIdleFudge = cMsTimeout // 2, + fnProcessEvents = self.oTstDrv.processPendingEvents); + if oTxsSession is None: + return False; + + # Wait for the connect task to time out. + self.oTstDrv.addTask(oTxsSession); + self.oTstDrv.processPendingEvents(); + oRc = self.oTstDrv.waitForTasks(cMsTimeout); + self.oTstDrv.removeTask(oTxsSession); + if oRc != oTxsSession: + if oRc is not None: + reporter.log('oRc=%s, expected %s' % (oRc, oTxsSession)); + self.oTstDrv.processPendingEvents(); + oTxsSession.cancelTask(); # this is synchronous + return False; + + # Check the status. + reporter.log2('TxsSession is ready, isSuccess() -> %s.' % (oTxsSession.isSuccess(),)); + if not oTxsSession.isSuccess(): + return False; + + reporter.log2('Disconnecting from TXS...'); + return oTxsSession.syncDisconnect(); + + + +class TxsConnectTask(TdTaskBase): + """ + Class that takes care of connecting to a VM. + """ + + class TxsConnectTaskVBoxCallback(vbox.VirtualBoxEventHandlerBase): + """ Class for looking for IPv4 address changes on interface 0.""" + def __init__(self, dArgs): + vbox.VirtualBoxEventHandlerBase.__init__(self, dArgs); + self.oParentTask = dArgs['oParentTask']; + self.sMachineId = dArgs['sMachineId']; + + def onGuestPropertyChange(self, sMachineId, sName, sValue, sFlags, fWasDeleted): + """Look for IP address.""" + reporter.log2('onGuestPropertyChange(,%s,%s,%s,%s,%s)' % (sMachineId, sName, sValue, sFlags, fWasDeleted)); + if sMachineId == self.sMachineId \ + and sName == '/VirtualBox/GuestInfo/Net/0/V4/IP': + oParentTask = self.oParentTask; + if oParentTask: + oParentTask._setIp(sValue); # pylint: disable=protected-access + + + def __init__(self, oSession, cMsTimeout, sIpAddr, sMacAddr, oIDhcpServer, fReversedSetup, fnProcessEvents = None): + TdTaskBase.__init__(self, utils.getCallerName(), fnProcessEvents = fnProcessEvents); + self.cMsTimeout = cMsTimeout; + self.fnProcessEvents = fnProcessEvents; + self.sIpAddr = None; + self.sNextIpAddr = None; + self.sMacAddr = sMacAddr; + self.oIDhcpServer = oIDhcpServer; + self.fReversedSetup = fReversedSetup; + self.oVBoxEventHandler = None; + self.oTxsSession = None; + + # Check that the input makes sense: + if (sMacAddr is None) != (oIDhcpServer is None) \ + or (sMacAddr and fReversedSetup) \ + or (sMacAddr and sIpAddr): + reporter.error('TxsConnectTask sMacAddr=%s oIDhcpServer=%s sIpAddr=%s fReversedSetup=%s' + % (sMacAddr, oIDhcpServer, sIpAddr, fReversedSetup,)); + raise base.GenError(); + + reporter.log2('TxsConnectTask: sIpAddr=%s fReversedSetup=%s' % (sIpAddr, fReversedSetup)) + if fReversedSetup is True: + self._openTcpSession(sIpAddr, fReversedSetup = True); + elif sIpAddr is not None and sIpAddr.strip() != '': + self._openTcpSession(sIpAddr, cMsIdleFudge = 5000); + else: + # + # If we've got no IP address, register callbacks that listens for + # the primary network adaptor of the VM to set a IPv4 guest prop. + # Note! The order in which things are done here is kind of important. + # + + # 0. The caller zaps the property before starting the VM. + #try: + # oSession.delGuestPropertyValue('/VirtualBox/GuestInfo/Net/0/V4/IP'); + #except: + # reporter.logXcpt(); + + # 1. Register the callback / event listener object. + dArgs = {'oParentTask':self, 'sMachineId':oSession.o.machine.id}; + self.oVBoxEventHandler = oSession.oVBox.registerDerivedEventHandler(self.TxsConnectTaskVBoxCallback, dArgs); + + # 2. Query the guest properties. + try: + sIpAddr = oSession.getGuestPropertyValue('/VirtualBox/GuestInfo/Net/0/V4/IP'); + except: + reporter.errorXcpt('IMachine::getGuestPropertyValue("/VirtualBox/GuestInfo/Net/0/V4/IP") failed'); + self._deregisterEventHandler(); + raise; + else: + if sIpAddr is not None: + self._setIp(sIpAddr); + + # + # If the network adapter of the VM is host-only we can talk poll IDHCPServer + # for the guest IP, allowing us to detect it for VMs without guest additions. + # This will when we're polled. + # + if sMacAddr is not None: + assert self.oIDhcpServer is not None; + + + # end __init__ + + def __del__(self): + """ Make sure we deregister the callback. """ + self._deregisterEventHandler(); + return TdTaskBase.__del__(self); + + def toString(self): + return '<%s cMsTimeout=%s, sIpAddr=%s, sNextIpAddr=%s, sMacAddr=%s, fReversedSetup=%s,' \ + ' oTxsSession=%s oVBoxEventHandler=%s>' \ + % (TdTaskBase.toString(self), self.cMsTimeout, self.sIpAddr, self.sNextIpAddr, self.sMacAddr, self.fReversedSetup, + self.oTxsSession, self.oVBoxEventHandler); + + def _deregisterEventHandler(self): + """Deregisters the event handler.""" + fRc = True; + oVBoxEventHandler = self.oVBoxEventHandler; + if oVBoxEventHandler is not None: + self.oVBoxEventHandler = None; + fRc = oVBoxEventHandler.unregister(); + oVBoxEventHandler.oParentTask = None; # Try avoid cylic deps. + return fRc; + + def _setIp(self, sIpAddr, fInitCall = False): + """Called when we get an IP. Will create a TXS session and signal the task.""" + sIpAddr = sIpAddr.strip(); + + if sIpAddr is not None \ + and sIpAddr != '': + if vbox.isIpAddrValid(sIpAddr) or fInitCall: + try: + for s in sIpAddr.split('.'): + i = int(s); + if str(i) != s: + raise Exception(); + except: + reporter.fatalXcpt(); + else: + reporter.log('TxsConnectTask: opening session to ip "%s"' % (sIpAddr)); + self._openTcpSession(sIpAddr, cMsIdleFudge = 5000); + return None; + + reporter.log('TxsConnectTask: Ignoring Bad ip "%s"' % (sIpAddr)); + else: + reporter.log2('TxsConnectTask: Ignoring empty ip "%s"' % (sIpAddr)); + return None; + + def _openTcpSession(self, sIpAddr, uPort = None, fReversedSetup = False, cMsIdleFudge = 0): + """ + Calls txsclient.openTcpSession and switches our task to reflect the + state of the subtask. + """ + self.oCv.acquire(); + if self.oTxsSession is None: + reporter.log2('_openTcpSession: sIpAddr=%s, uPort=%d, fReversedSetup=%s' % + (sIpAddr, uPort if uPort is not None else 0, fReversedSetup)); + self.sIpAddr = sIpAddr; + self.oTxsSession = txsclient.openTcpSession(self.cMsTimeout, sIpAddr, uPort, fReversedSetup, + cMsIdleFudge, fnProcessEvents = self.fnProcessEvents); + self.oTxsSession.setTaskOwner(self); + else: + self.sNextIpAddr = sIpAddr; + reporter.log2('_openTcpSession: sNextIpAddr=%s' % (sIpAddr,)); + self.oCv.release(); + return None; + + def notifyAboutReadyTask(self, oTxsSession): + """ + Called by the TXS session task when it's done. + + We'll signal the task completed or retry depending on the result. + """ + + self.oCv.acquire(); + + # Disassociate ourselves with the session (avoid cyclic ref) + oTxsSession.setTaskOwner(None); + fSuccess = oTxsSession.isSuccess(); + if self.oTxsSession is not None: + if not fSuccess: + self.oTxsSession = None; + if fSuccess and self.fReversedSetup: + self.sIpAddr = oTxsSession.oTransport.sHostname; + else: + fSuccess = False; + + # Signal done, or retry? + fDeregister = False; + if fSuccess \ + or self.fReversedSetup \ + or self.getAgeAsMs() >= self.cMsTimeout: + self.signalTaskLocked(); + fDeregister = True; + else: + sIpAddr = self.sNextIpAddr if self.sNextIpAddr is not None else self.sIpAddr; + self._openTcpSession(sIpAddr, cMsIdleFudge = 5000); + + self.oCv.release(); + + # If we're done, deregister the callback (w/o owning lock). It will + if fDeregister: + self._deregisterEventHandler(); + return True; + + def _pollDhcpServer(self): + """ + Polls the DHCP server by MAC address in host-only setups. + """ + + if self.sIpAddr: + return False; + + if self.oIDhcpServer is None or not self.sMacAddr: + return False; + + try: + (sIpAddr, sState, secIssued, secExpire) = self.oIDhcpServer.findLeaseByMAC(self.sMacAddr, 0); + except: + reporter.log4Xcpt('sMacAddr=%s' % (self.sMacAddr,)); + return False; + + secNow = utils.secondsSinceUnixEpoch(); + reporter.log2('dhcp poll: secNow=%s secExpire=%s secIssued=%s sState=%s sIpAddr=%s' + % (secNow, secExpire, secIssued, sState, sIpAddr,)); + if secNow > secExpire or sState != 'acked' or not sIpAddr: + return False; + + reporter.log('dhcp poll: sIpAddr=%s secExpire=%s (%s TTL) secIssued=%s (%s ago)' + % (sIpAddr, secExpire, secExpire - secNow, secIssued, secNow - secIssued,)); + self._setIp(sIpAddr); + return True; + + # + # Task methods + # + + def pollTask(self, fLocked = False): + """ + Overridden pollTask method. + """ + self._pollDhcpServer(); + return TdTaskBase.pollTask(self, fLocked); + + # + # Public methods + # + + def getResult(self): + """ + Returns the connected TXS session object on success. + Returns None on failure or if the task has not yet completed. + """ + self.oCv.acquire(); + oTxsSession = self.oTxsSession; + self.oCv.release(); + + if oTxsSession is not None and not oTxsSession.isSuccess(): + oTxsSession = None; + return oTxsSession; + + def cancelTask(self): + """ Cancels the task. """ + self._deregisterEventHandler(); # (make sure to avoid cyclic fun) + self.oCv.acquire(); + if not self.fSignalled: + oTxsSession = self.oTxsSession; + if oTxsSession is not None: + self.oCv.release(); + oTxsSession.setTaskOwner(None); + oTxsSession.cancelTask(); + oTxsSession.waitForTask(1000); + self.oCv.acquire(); + self.signalTaskLocked(); + self.oCv.release(); + return True; + + + +class AdditionsStatusTask(TdTaskBase): + """ + Class that takes care of waiting till the guest additions are in a given state. + """ + + class AdditionsStatusTaskCallback(vbox.EventHandlerBase): + """ Class for looking for IPv4 address changes on interface 0.""" + def __init__(self, dArgs): + self.oParentTask = dArgs['oParentTask']; + vbox.EventHandlerBase.__init__(self, dArgs, self.oParentTask.oSession.fpApiVer, + 'AdditionsStatusTaskCallback/%s' % (self.oParentTask.oSession.sName,)); + + def handleEvent(self, oEvt): + try: + enmType = oEvt.type; + except: + reporter.errorXcpt(); + else: + reporter.log2('AdditionsStatusTaskCallback:handleEvent: enmType=%s' % (enmType,)); + if enmType == vboxcon.VBoxEventType_OnGuestAdditionsStatusChanged: + oParentTask = self.oParentTask; + if oParentTask: + oParentTask.pollTask(); + + # end + + + def __init__(self, oSession, oIGuest, cMsTimeout = 120000, aenmWaitForRunLevels = None, aenmWaitForActive = None, + aenmWaitForInactive = None): + """ + aenmWaitForRunLevels - List of run level values to wait for (success if one matches). + aenmWaitForActive - List facilities (type values) that must be active. + aenmWaitForInactive - List facilities (type values) that must be inactive. + + The default is to wait for AdditionsRunLevelType_Userland if all three lists + are unspecified or empty. + """ + TdTaskBase.__init__(self, utils.getCallerName()); + self.oSession = oSession # type: vboxwrappers.SessionWrapper + self.oIGuest = oIGuest; + self.cMsTimeout = cMsTimeout; + self.fSucceeded = False; + self.oVBoxEventHandler = None; + self.aenmWaitForRunLevels = aenmWaitForRunLevels if aenmWaitForRunLevels else []; + self.aenmWaitForActive = aenmWaitForActive if aenmWaitForActive else []; + self.aenmWaitForInactive = aenmWaitForInactive if aenmWaitForInactive else []; + + # Provide a sensible default if nothing is given. + if not self.aenmWaitForRunLevels and not self.aenmWaitForActive and not self.aenmWaitForInactive: + self.aenmWaitForRunLevels = [vboxcon.AdditionsRunLevelType_Userland,]; + + # Register the event handler on hosts which has it: + if oSession.fpApiVer >= 6.1 or hasattr(vboxcon, 'VBoxEventType_OnGuestAdditionsStatusChanged'): + aenmEvents = (vboxcon.VBoxEventType_OnGuestAdditionsStatusChanged,); + dArgs = { + 'oParentTask': self, + }; + self.oVBoxEventHandler = vbox.EventHandlerBase.registerDerivedEventHandler(oSession.oVBoxMgr, + oSession.fpApiVer, + self.AdditionsStatusTaskCallback, + dArgs, + oIGuest, + 'IGuest', + 'AdditionsStatusTaskCallback', + aenmEvents = aenmEvents); + reporter.log2('AdditionsStatusTask: %s' % (self.toString(), )); + + def __del__(self): + """ Make sure we deregister the callback. """ + self._deregisterEventHandler(); + self.oIGuest = None; + return TdTaskBase.__del__(self); + + def toString(self): + return '<%s cMsTimeout=%s, fSucceeded=%s, aenmWaitForRunLevels=%s, aenmWaitForActive=%s, aenmWaitForInactive=%s, ' \ + 'oVBoxEventHandler=%s>' \ + % (TdTaskBase.toString(self), self.cMsTimeout, self.fSucceeded, self.aenmWaitForRunLevels, self.aenmWaitForActive, + self.aenmWaitForInactive, self.oVBoxEventHandler,); + + def _deregisterEventHandler(self): + """Deregisters the event handler.""" + fRc = True; + oVBoxEventHandler = self.oVBoxEventHandler; + if oVBoxEventHandler is not None: + self.oVBoxEventHandler = None; + fRc = oVBoxEventHandler.unregister(); + oVBoxEventHandler.oParentTask = None; # Try avoid cylic deps. + return fRc; + + def _poll(self): + """ + Internal worker for pollTask() that returns the new signalled state. + """ + + # + # Check if any of the runlevels we wait for have been reached: + # + if self.aenmWaitForRunLevels: + try: + enmRunLevel = self.oIGuest.additionsRunLevel; + except: + reporter.errorXcpt(); + return True; + if enmRunLevel not in self.aenmWaitForRunLevels: + reporter.log6('AdditionsStatusTask/poll: enmRunLevel=%s not in %s' % (enmRunLevel, self.aenmWaitForRunLevels,)); + return False; + reporter.log2('AdditionsStatusTask/poll: enmRunLevel=%s matched %s!' % (enmRunLevel, self.aenmWaitForRunLevels,)); + + + # + # Check for the facilities that must all be active. + # + for enmFacility in self.aenmWaitForActive: + try: + (enmStatus, _) = self.oIGuest.getFacilityStatus(enmFacility); + except: + reporter.errorXcpt('enmFacility=%s' % (enmFacility,)); + return True; + if enmStatus != vboxcon.AdditionsFacilityStatus_Active: + reporter.log2('AdditionsStatusTask/poll: enmFacility=%s not active: %s' % (enmFacility, enmStatus,)); + return False; + + # + # Check for the facilities that must all be inactive or terminated. + # + for enmFacility in self.aenmWaitForInactive: + try: + (enmStatus, _) = self.oIGuest.getFacilityStatus(enmFacility); + except: + reporter.errorXcpt('enmFacility=%s' % (enmFacility,)); + return True; + if enmStatus not in (vboxcon.AdditionsFacilityStatus_Inactive, + vboxcon.AdditionsFacilityStatus_Terminated): + reporter.log2('AdditionsStatusTask/poll: enmFacility=%s not inactive: %s' % (enmFacility, enmStatus,)); + return False; + + + reporter.log('AdditionsStatusTask: Poll succeeded, signalling...'); + self.fSucceeded = True; + return True; + + + # + # Task methods + # + + def pollTask(self, fLocked = False): + """ + Overridden pollTask method. + """ + if not fLocked: + self.lockTask(); + + fDeregister = False; + fRc = self.fSignalled; + if not fRc: + fRc = self._poll(); + if fRc or self.getAgeAsMs() >= self.cMsTimeout: + self.signalTaskLocked(); + fDeregister = True; + + if not fLocked: + self.unlockTask(); + + # If we're done, deregister the event callback (w/o owning lock). + if fDeregister: + self._deregisterEventHandler(); + return fRc; + + def getResult(self): + """ + Returns true if the we succeeded. + Returns false if not. If the task is signalled already, then we + encountered a problem while polling. + """ + return self.fSucceeded; + + def cancelTask(self): + """ + Cancels the task. + Just to actively disengage the event handler. + """ + self._deregisterEventHandler(); + return True; + diff --git a/src/VBox/ValidationKit/testdriver/win-vbox-net-drvstore-cleanup.ps1 b/src/VBox/ValidationKit/testdriver/win-vbox-net-drvstore-cleanup.ps1 new file mode 100644 index 00000000..054fc20c --- /dev/null +++ b/src/VBox/ValidationKit/testdriver/win-vbox-net-drvstore-cleanup.ps1 @@ -0,0 +1,71 @@ +# $Id: win-vbox-net-drvstore-cleanup.ps1 $ +## @file +# VirtualBox Validation Kit - network cleanup script (powershell). +# + +# +# Copyright (C) 2006-2022 Oracle and/or its affiliates. +# +# This file is part of VirtualBox base platform packages, as +# available from https://www.virtualbox.org. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation, in version 3 of the +# License. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see <https://www.gnu.org/licenses>. +# +# The contents of this file may alternatively be used under the terms +# of the Common Development and Distribution License Version 1.0 +# (CDDL), a copy of it is provided in the "COPYING.CDDL" file included +# in the VirtualBox distribution, in which case the provisions of the +# CDDL are applicable instead of those of the GPL. +# +# You may elect to license modified versions of this file under the +# terms and conditions of either the GPL or the CDDL or both. +# +# SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +# + +param([switch]$confirm) + +Function AskForConfirmation ($title_text, $message_text, $yes_text, $no_text) +{ + if ($confirm) { + $title = $title_text + $message = $message_text + + $yes = New-Object System.Management.Automation.Host.ChoiceDescription "&Yes", $yes_text + + $no = New-Object System.Management.Automation.Host.ChoiceDescription "&No", $no_text + + $options = [System.Management.Automation.Host.ChoiceDescription[]]($yes, $no) + + $result = $host.ui.PromptForChoice($title, $message, $options, 0) + } else { + $result = 0 + } + + return $result +} + +pnputil -e | ForEach-Object { if ($_ -match "Published name :.*(oem\d+\.inf)") {$inf=$matches[1]} elseif ($_ -match "Driver package provider :.*Oracle") {$inf + " " + $_} } + +$result = AskForConfirmation "Clean up the driver store" ` + "Do you want to delete all VirtualBox drivers from the driver store?" ` + "Deletes all VirtualBox drivers from the driver store." ` + "No modifications to the driver store will be made." + +switch ($result) + { + 0 {pnputil -e | ForEach-Object { if ($_ -match "Published name :.*(oem\d+\.inf)") {$inf=$matches[1]} elseif ($_ -match "Driver package provider :.*Oracle") {$inf} } | ForEach-Object { pnputil -d $inf } } + 1 {"Removal cancelled."} + } + diff --git a/src/VBox/ValidationKit/testdriver/win-vbox-net-uninstall.ps1 b/src/VBox/ValidationKit/testdriver/win-vbox-net-uninstall.ps1 new file mode 100644 index 00000000..8bef4731 --- /dev/null +++ b/src/VBox/ValidationKit/testdriver/win-vbox-net-uninstall.ps1 @@ -0,0 +1,253 @@ +# $Id: win-vbox-net-uninstall.ps1 $ +## @file +# VirtualBox Validation Kit - network cleanup script (powershell). +# + +# +# Copyright (C) 2006-2022 Oracle and/or its affiliates. +# +# This file is part of VirtualBox base platform packages, as +# available from https://www.virtualbox.org. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation, in version 3 of the +# License. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see <https://www.gnu.org/licenses>. +# +# The contents of this file may alternatively be used under the terms +# of the Common Development and Distribution License Version 1.0 +# (CDDL), a copy of it is provided in the "COPYING.CDDL" file included +# in the VirtualBox distribution, in which case the provisions of the +# CDDL are applicable instead of those of the GPL. +# +# You may elect to license modified versions of this file under the +# terms and conditions of either the GPL or the CDDL or both. +# +# SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +# + +param([switch]$confirm) + +Function AskForConfirmation ($title_text, $message_text, $yes_text, $no_text) +{ + if ($confirm) { + $title = $title_text + $message = $message_text + + $yes = New-Object System.Management.Automation.Host.ChoiceDescription "&Yes", $yes_text + + $no = New-Object System.Management.Automation.Host.ChoiceDescription "&No", $no_text + + $options = [System.Management.Automation.Host.ChoiceDescription[]]($yes, $no) + + $result = $host.ui.PromptForChoice($title, $message, $options, 0) + } else { + $result = 0 + } + + return $result +} + +Function DeleteUnmatchingKeys ($title_text, $reg_key) +{ + $ghostcon = @(Get-ChildItem ($reg_key) | Where-Object { !$connections.ContainsKey($_.PSChildName) } ) + if ($ghostcon.count -eq 0) { + Write-Host "`nNo ghost connections has been found -- nothing to do" + } else { + Write-Host "`nParameter keys for the following connections will be removed:" + Write-Host ($ghostcon | Out-String) + + $result = AskForConfirmation $title_text ` + "Do you want to delete the keys listed above?" ` + "Deletes all ghost connection keys from the registry." ` + "No modifications to the registry will be made." + + switch ($result) + { + 0 {$ghostcon.GetEnumerator() | ForEach-Object { Remove-Item -Path $_ -Recurse }} + 1 {"Removal cancelled."} + } + } +} + + +Push-Location +cd "Registry::" +Write-Host "Retrieving valid connections:" +$iftypes = @{} +$connections = @{} +$ghostcon_names = @{} +Get-Item ".\HKLM\SYSTEM\CurrentControlSet\Control\Class\{4D36E972-E325-11CE-BFC1-08002BE10318}\0*" | ` + ForEach-Object { + $prop = (Get-ItemProperty $_.PSPath) + $conn = $null + if (Test-Path ("HKLM\SYSTEM\CurrentControlSet\Control\Network\{4D36E972-E325-11CE-BFC1-08002BE10318}\" + $prop.NetCfgInstanceId + "\Connection")) { + $conn = (Get-ItemProperty ("HKLM\SYSTEM\CurrentControlSet\Control\Network\{4D36E972-E325-11CE-BFC1-08002BE10318}\" + $prop.NetCfgInstanceId + "\Connection")) + } + $iftype = $prop."*IfType" + if ($iftypes.ContainsKey($iftype)) { + $iftypes[$iftype] = $iftypes[$iftype] + [Math]::pow(2,$prop.NetLuidIndex) + } else { + $iftypes[$iftype] = [Math]::pow(2,$prop.NetLuidIndex) + } + if ($conn -ne $null) { + $connections[$prop.NetCfgInstanceId] = $conn.Name + Write-Host $prop.NetCfgInstanceId $conn.Name "|" $prop."*IfType" $prop.NetLuidIndex $prop.DriverDesc + } else { + Write-Host $prop.NetCfgInstanceId [MISSING] "|" $prop."*IfType" $prop.NetLuidIndex $prop.DriverDesc + } + } + +# Someday we may want to process other types than Ethernet as well: $iftypes.GetEnumerator() | ForEach-Object { +if ($iftypes[6] -gt 9223372036854775808) { + Write-Host "Found more than 63 interfaces (mask=" $iftypes[6] ") -- bailing out" + exit +} +Write-Host "`nChecking if the used LUID index mask is correct:" +$correctmask = [BitConverter]::GetBytes([int64]($iftypes[6])) +$actualmask = (Get-ItemProperty -Path "HKLM\SYSTEM\CurrentControlSet\Services\NDIS\IfTypes\6" -Name "IfUsedNetLuidIndices").IfUsedNetLuidIndices +$needcorrection = $FALSE +$ai = 0 +$lastnonzero = 0 +for ($ci = 0; $ci -lt $correctmask.Length; $ci++) { + if ($ai -lt $actualmask.Length) { + $aval = $actualmask[$ai++] + } else { + $aval = 0 + } + if ($correctmask[$ci] -ne 0) { + $lastnonzero = $ci + } + if ($correctmask[$ci] -eq $aval) { + Write-Host "DEBUG: " $correctmask[$ci].ToString("X2") " == " $aval.ToString("X2") + } else { + Write-Host "DEBUG: " $correctmask[$ci].ToString("X2") " != " $aval.ToString("X2") + $needcorrection = $TRUE + } +} +if ($ai -lt $actualmask.Length) { + for (; $ai -lt $actualmask.Length; $ai++) { + if ($actualmask[$ai] -eq 0) { + Write-Host "DEBUG: 0 == 0" + } else { + Write-Host "DEBUG: " $actualmask[$ai].ToString("X2") " != 0" + $needcorrection = $TRUE + } + } +} +if ($needcorrection) { + Write-Host "Current mask is " ($actualmask|foreach {$_.ToString("X2")}) ", while it should be" ($correctmask|foreach {$_.ToString("X2")}) + if ($confirm) { + Set-ItemProperty -Path "HKLM\SYSTEM\CurrentControlSet\Services\NDIS\IfTypes\6" -Name "IfUsedNetLuidIndices" -Value $correctmask -Type Binary -Confirm + } else { + Set-ItemProperty -Path "HKLM\SYSTEM\CurrentControlSet\Services\NDIS\IfTypes\6" -Name "IfUsedNetLuidIndices" -Value $correctmask -Type Binary + } +} else { + Write-Host "The used LUID index mask is correct -- nothing to do" +} + +#Write-Host ($connections | Out-String) +$ghostcon = @(Get-ChildItem ("HKLM\SYSTEM\CurrentControlSet\Control\Network\{4D36E972-E325-11CE-BFC1-08002BE10318}") | Where-Object { !$connections.ContainsKey($_.PSChildName) -and $_.PSChildName -ne "Descriptions" } ) +if ($ghostcon -eq $null) { + Write-Host "`nNo ghost connections has been found -- nothing to do" +} else { + Write-Host "`nThe following connections will be removed:" + #Write-Host ($ghostcon | Out-String) + + $ghostcon.GetEnumerator() | ForEach-Object { + $prop = (Get-ItemProperty "$_\Connection") + if ($prop.PnPInstanceId -eq $null) { + Write-Host "WARNING! PnPInstanceId does not exist for" $_.PSChildName + } elseif (!($prop.PnPInstanceId.ToString() -match "SUN_VBOXNETFLTMP")) { + Write-Host "WARNING! PnPInstanceId (" $prop.PnPInstanceId.ToString() ") does not match ROOT\SUN_VBOXNETFLTMP for" $_.PSChildName + } + if ($prop.Name -eq $null) { + Write-Host "WARNING! Name does not exist for" $_.PSChildName + } else { + $ghostcon_names.Add($_.PSChildName, $prop.Name) + Write-Host $_.PSChildName -nonewline + Write-Host " " -nonewline + Write-Host $prop.Name + } + } + + $result = AskForConfirmation "Delete Registry Keys" ` + "Do you want to delete the keys listed above?" ` + "Deletes all ghost connection keys from the registry." ` + "No modifications to the registry will be made." + + switch ($result) + { + 0 {$ghostcon.GetEnumerator() | ForEach-Object { Remove-Item -Path $_.PSPath -Recurse }} + 1 {"Removal cancelled."} + } +} + +# Delete WFPLWFS parameter keys +DeleteUnmatchingKeys "Delete WFPLWFS Parameter Keys (Adapter subkey)" "HKLM\SYSTEM\CurrentControlSet\Services\WFPLWFS\Parameters\Adapters" +DeleteUnmatchingKeys "Delete WFPLWFS Parameter Keys (NdisAdapter subkey)" "HKLM\SYSTEM\CurrentControlSet\Services\WFPLWFS\Parameters\NdisAdapters" +# Delete Psched parameter keys +DeleteUnmatchingKeys "Delete Psched Parameter Keys (Adapter subkey)" "HKLM\SYSTEM\CurrentControlSet\Services\Psched\Parameters\Adapters" +DeleteUnmatchingKeys "Delete Psched Parameter Keys (NdisAdapter subkey)" "HKLM\SYSTEM\CurrentControlSet\Services\Psched\Parameters\NdisAdapters" + +# Clean up NSI entries +$nsi_obsolete = New-Object System.Collections.ArrayList +$nsi_path = "HKLM\SYSTEM\CurrentControlSet\Control\Nsi\{EB004A11-9B1A-11D4-9123-0050047759BC}\10" +$nsi = (Get-Item $nsi_path) | Select-Object -ExpandProperty property +$nsi | ForEach-Object { + $value = (Get-ItemProperty -Path $nsi_path -Name $_).$_ + [byte[]]$guid_bytes = $value[1040..1055] + $guid = New-Object -TypeName System.Guid -ArgumentList (,$guid_bytes) + $guid_string = $guid.ToString("B").ToUpper() + $nsi_conn_name_last = 6 + $value[4] + $value[5]*256 + $nsi_conn_name = [Text.Encoding]::Unicode.GetString($value[6..$nsi_conn_name_last]) + $nsi_if_name_last = 522 + $value[520] + $value[521]*256 + $nsi_if_name = [Text.Encoding]::Unicode.GetString($value[522..$nsi_if_name_last]) + Write-Host $_ -nonewline + Write-Host " " -nonewline + Write-Host $guid_string -nonewline + Write-Host " " -nonewline + if ($connections.ContainsKey($guid_string)) { + Write-Host $nsi_if_name + } else { + [void] $nsi_obsolete.Add($_) + Write-Host "[OBSOLETE] " $nsi_if_name -foregroundcolor red + } +} + +$result = AskForConfirmation "Delete NSI Entries" ` + "Do you want to delete the entries marked in red above?" ` + "Deletes all marked entries from the NSI registry key." ` + "No modifications to the registry will be made." + +switch ($result) + { + 0 {$nsi_obsolete.GetEnumerator() | ForEach-Object { Remove-ItemProperty -Path $nsi_path -Name $_ }} + 1 {"Removal cancelled."} + } + +# Clean up uninstalled connections +if ( (Get-ChildItem "HKLM\SYSTEM\CurrentControlSet\Control\Network\Uninstalled" | Measure-Object).Count -gt 10 ) { + $result = AskForConfirmation "Delete Uninstalled Network Connection Registry Keys" ` + "There are over 10 uninstalled network connections accumulated in the registry. Do you want to delete them?" ` + "Deletes uninstalled connection keys from the registry." ` + "No modifications to the registry will be made." + + switch ($result) + { + 0 {Remove-Item -Path "HKLM\SYSTEM\CurrentControlSet\Control\Network\Uninstalled\*" -Recurse} + 1 {"Removal cancelled."} + } +} else { + Write-Host "Less than 10 uninstalled connections -- no action yet required." +} + +Pop-Location diff --git a/src/VBox/ValidationKit/testdriver/winbase.py b/src/VBox/ValidationKit/testdriver/winbase.py new file mode 100755 index 00000000..92deac1d --- /dev/null +++ b/src/VBox/ValidationKit/testdriver/winbase.py @@ -0,0 +1,336 @@ +# -*- coding: utf-8 -*- +# $Id: winbase.py $ + +""" +This module is here to externalize some Windows specifics that gives pychecker +a hard time when running on non-Windows systems. +""" + +__copyright__ = \ +""" +Copyright (C) 2010-2022 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 153224 $" + + +# Standard Python imports. +import ctypes; +import os; +import sys; + +# Windows specific imports. +import pywintypes; # pylint: disable=import-error +import winerror; # pylint: disable=import-error +import win32con; # pylint: disable=import-error +import win32api; # pylint: disable=import-error +import win32console; # pylint: disable=import-error +import win32event; # pylint: disable=import-error +import win32process; # pylint: disable=import-error + +# Validation Kit imports. +from testdriver import reporter; + +# Python 3 hacks: +if sys.version_info[0] >= 3: + long = int; # pylint: disable=redefined-builtin,invalid-name + + +# +# Windows specific implementation of base functions. +# + +def processInterrupt(uPid): + """ + The Windows version of base.processInterrupt + + Note! This doesn't work terribly well with a lot of processes. + """ + try: + # pylint: disable=no-member + win32console.GenerateConsoleCtrlEvent(win32con.CTRL_BREAK_EVENT, uPid); # pylint: disable=c-extension-no-member + #GenerateConsoleCtrlEvent = ctypes.windll.kernel32.GenerateConsoleCtrlEvent + #rc = GenerateConsoleCtrlEvent(1, uPid); + #reporter.log('GenerateConsoleCtrlEvent -> %s' % (rc,)); + fRc = True; + except: + reporter.logXcpt('uPid=%s' % (uPid,)); + fRc = False; + return fRc; + +def postThreadMesssageClose(uTid): + """ Posts a WM_CLOSE message to the specified thread.""" + fRc = False; + try: + win32api.PostThreadMessage(uTid, win32con.WM_CLOSE, 0, 0); # pylint: disable=no-member,c-extension-no-member + fRc = True; + except: + reporter.logXcpt('uTid=%s' % (uTid,)); + return fRc; + +def postThreadMesssageQuit(uTid): + """ Posts a WM_QUIT message to the specified thread.""" + fRc = False; + try: + win32api.PostThreadMessage(uTid, win32con.WM_QUIT, # pylint: disable=no-member,c-extension-no-member + 0x40010004, 0); # DBG_TERMINATE_PROCESS + fRc = True; + except: + reporter.logXcpt('uTid=%s' % (uTid,)); + return fRc; + +def processTerminate(uPid): + """ The Windows version of base.processTerminate """ + # pylint: disable=no-member + fRc = False; + try: + hProcess = win32api.OpenProcess(win32con.PROCESS_TERMINATE, # pylint: disable=no-member,c-extension-no-member + False, uPid); + except: + reporter.logXcpt('uPid=%s' % (uPid,)); + else: + try: + win32process.TerminateProcess(hProcess, # pylint: disable=no-member,c-extension-no-member + 0x40010004); # DBG_TERMINATE_PROCESS + fRc = True; + except: + reporter.logXcpt('uPid=%s' % (uPid,)); + hProcess.Close(); #win32api.CloseHandle(hProcess) + return fRc; + +def processKill(uPid): + """ The Windows version of base.processKill """ + return processTerminate(uPid); + +def processExists(uPid): + """ The Windows version of base.processExists """ + # We try open the process for waiting since this is generally only forbidden in a very few cases. + try: + hProcess = win32api.OpenProcess(win32con.SYNCHRONIZE, False, uPid); # pylint: disable=no-member,c-extension-no-member + except pywintypes.error as oXcpt: # pylint: disable=no-member + if oXcpt.winerror == winerror.ERROR_INVALID_PARAMETER: + return False; + if oXcpt.winerror != winerror.ERROR_ACCESS_DENIED: + reporter.logXcpt('uPid=%s oXcpt=%s' % (uPid, oXcpt)); + return False; + reporter.logXcpt('uPid=%s oXcpt=%s' % (uPid, oXcpt)); + except Exception as oXcpt: + reporter.logXcpt('uPid=%s' % (uPid,)); + return False; + else: + hProcess.Close(); #win32api.CloseHandle(hProcess) + return True; + +def processCheckPidAndName(uPid, sName): + """ The Windows version of base.processCheckPidAndName """ + fRc = processExists(uPid); + if fRc is True: + try: + from win32com.client import GetObject; # pylint: disable=import-error + oWmi = GetObject('winmgmts:'); + aoProcesses = oWmi.InstancesOf('Win32_Process'); + for oProcess in aoProcesses: + if long(oProcess.Properties_("ProcessId").Value) == uPid: + sCurName = oProcess.Properties_("Name").Value; + reporter.log2('uPid=%s sName=%s sCurName=%s' % (uPid, sName, sCurName)); + sName = sName.lower(); + sCurName = sCurName.lower(); + if os.path.basename(sName) == sName: + sCurName = os.path.basename(sCurName); + + if sCurName == sName \ + or sCurName + '.exe' == sName \ + or sCurName == sName + '.exe': + fRc = True; + break; + except: + reporter.logXcpt('uPid=%s sName=%s' % (uPid, sName)); + return fRc; + +# +# Some helper functions. +# +def processCreate(sName, asArgs): + """ + Returns a (pid, handle, tid) tuple on success. (-1, None) on failure (logged). + """ + + # Construct a command line. + sCmdLine = ''; + for sArg in asArgs: + if sCmdLine == '': + sCmdLine += '"'; + else: + sCmdLine += ' "'; + sCmdLine += sArg; + sCmdLine += '"'; + + # Try start the process. + # pylint: disable=no-member + dwCreationFlags = win32con.CREATE_NEW_PROCESS_GROUP; + oStartupInfo = win32process.STARTUPINFO(); # pylint: disable=c-extension-no-member + try: + (hProcess, hThread, uPid, uTid) = win32process.CreateProcess(sName, # pylint: disable=c-extension-no-member + sCmdLine, # CommandLine + None, # ProcessAttributes + None, # ThreadAttibutes + 1, # fInheritHandles + dwCreationFlags, + None, # Environment + None, # CurrentDirectory. + oStartupInfo); + except: + reporter.logXcpt('sName="%s" sCmdLine="%s"' % (sName, sCmdLine)); + return (-1, None, -1); + + # Dispense with the thread handle. + try: + hThread.Close(); # win32api.CloseHandle(hThread); + except: + reporter.logXcpt(); + + # Try get full access to the process. + try: + hProcessFullAccess = win32api.DuplicateHandle( # pylint: disable=c-extension-no-member + win32api.GetCurrentProcess(), # pylint: disable=c-extension-no-member + hProcess, + win32api.GetCurrentProcess(), # pylint: disable=c-extension-no-member + win32con.PROCESS_TERMINATE + | win32con.PROCESS_QUERY_INFORMATION + | win32con.SYNCHRONIZE + | win32con.DELETE, + False, + 0); + hProcess.Close(); # win32api.CloseHandle(hProcess); + hProcess = hProcessFullAccess; + except: + reporter.logXcpt(); + reporter.log2('processCreate -> %#x, hProcess=%s %#x' % (uPid, hProcess, hProcess.handle,)); + return (uPid, hProcess, uTid); + +def processPollByHandle(hProcess): + """ + Polls the process handle to see if it has finished (True) or not (False). + """ + try: + dwWait = win32event.WaitForSingleObject(hProcess, 0); # pylint: disable=no-member,c-extension-no-member + except: + reporter.logXcpt('hProcess=%s %#x' % (hProcess, hProcess.handle,)); + return True; + return dwWait != win32con.WAIT_TIMEOUT; #0x102; # + + +def processTerminateByHandle(hProcess): + """ + Terminates the process. + """ + try: + win32api.TerminateProcess(hProcess, # pylint: disable=no-member,c-extension-no-member + 0x40010004); # DBG_TERMINATE_PROCESS + except: + reporter.logXcpt('hProcess=%s %#x' % (hProcess, hProcess.handle,)); + return False; + return True; + +# +# Misc +# + +def logMemoryStats(): + """ + Logs windows memory stats. + """ + class MemoryStatusEx(ctypes.Structure): + """ MEMORYSTATUSEX """ + kaFields = [ + ( 'dwLength', ctypes.c_ulong ), + ( 'dwMemoryLoad', ctypes.c_ulong ), + ( 'ullTotalPhys', ctypes.c_ulonglong ), + ( 'ullAvailPhys', ctypes.c_ulonglong ), + ( 'ullTotalPageFile', ctypes.c_ulonglong ), + ( 'ullAvailPageFile', ctypes.c_ulonglong ), + ( 'ullTotalVirtual', ctypes.c_ulonglong ), + ( 'ullAvailVirtual', ctypes.c_ulonglong ), + ( 'ullAvailExtendedVirtual', ctypes.c_ulonglong ), + ]; + _fields_ = kaFields; # pylint: disable=invalid-name + + def __init__(self): + super(MemoryStatusEx, self).__init__(); + self.dwLength = ctypes.sizeof(self); + + try: + oStats = MemoryStatusEx(); + ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(oStats)); + except: + reporter.logXcpt(); + return False; + + reporter.log('Memory statistics:'); + for sField, _ in MemoryStatusEx.kaFields: + reporter.log(' %32s: %s' % (sField, getattr(oStats, sField))); + return True; + +def checkProcessHeap(): + """ + Calls HeapValidate(GetProcessHeap(), 0, NULL); + """ + + # Get the process heap. + try: + hHeap = ctypes.windll.kernel32.GetProcessHeap(); + except: + reporter.logXcpt(); + return False; + + # Check it. + try: + fIsOkay = ctypes.windll.kernel32.HeapValidate(hHeap, 0, None); + except: + reporter.logXcpt(); + return False; + + if fIsOkay == 0: + reporter.log('HeapValidate failed!'); + + # Try trigger a dump using c:\utils\procdump64.exe. + from common import utils; + + iPid = os.getpid(); + asArgs = [ 'e:\\utils\\procdump64.exe', '-ma', '%s' % (iPid,), 'c:\\CrashDumps\\python.exe-%u-heap.dmp' % (iPid,)]; + if utils.getHostArch() != 'amd64': + asArgs[0] = 'c:\\utils\\procdump.exe' + reporter.log('Trying to dump this process using: %s' % (asArgs,)); + utils.processCall(asArgs); + + # Generate a crash exception. + ctypes.windll.msvcrt.strcpy(None, None, 1024); + + return True; + |