diff options
Diffstat (limited to 'src/VBox/ValidationKit/testboxscript/testboxscript_real.py')
| -rwxr-xr-x | src/VBox/ValidationKit/testboxscript/testboxscript_real.py | 1073 |
1 files changed, 1073 insertions, 0 deletions
diff --git a/src/VBox/ValidationKit/testboxscript/testboxscript_real.py b/src/VBox/ValidationKit/testboxscript/testboxscript_real.py new file mode 100755 index 00000000..7a2581ae --- /dev/null +++ b/src/VBox/ValidationKit/testboxscript/testboxscript_real.py @@ -0,0 +1,1073 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# $Id: testboxscript_real.py $ + +""" +TestBox Script - main(). +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Standard python imports. +import math +import os +from optparse import OptionParser # pylint: disable=deprecated-module +import platform +import random +import shutil +import sys +import tempfile +import time +import uuid + +# Only the main script needs to modify the path. +try: __file__ +except: __file__ = sys.argv[0]; +g_ksTestScriptDir = os.path.dirname(os.path.abspath(__file__)); +g_ksValidationKitDir = os.path.dirname(g_ksTestScriptDir); +sys.path.extend([g_ksTestScriptDir, g_ksValidationKitDir]); + +# Validation Kit imports. +from common import constants; +from common import utils; +import testboxcommons; +from testboxcommons import TestBoxException; +from testboxcommand import TestBoxCommand; +from testboxconnection import TestBoxConnection; +from testboxscript import TBS_EXITCODE_SYNTAX, TBS_EXITCODE_FAILURE; + +# Python 3 hacks: +if sys.version_info[0] >= 3: + long = int; # pylint: disable=redefined-builtin,invalid-name + + +class TestBoxScriptException(Exception): + """ For raising exceptions during TestBoxScript.__init__. """ + pass; # pylint: disable=unnecessary-pass + + +class TestBoxScript(object): + """ + Implementation of the test box script. + Communicate with test manager and perform offered actions. + """ + + ## @name Class Constants. + # @{ + + # Scratch space round value (MB). + kcMbScratchSpaceRounding = 64 + # Memory size round value (MB). + kcMbMemoryRounding = 4 + # A NULL UUID in string form. + ksNullUuid = '00000000-0000-0000-0000-000000000000'; + # The minimum dispatch loop delay. + kcSecMinDelay = 12; + # The maximum dispatch loop delay (inclusive). + kcSecMaxDelay = 24; + # The minimum sign-on delay. + kcSecMinSignOnDelay = 30; + # The maximum sign-on delay (inclusive). + kcSecMaxSignOnDelay = 60; + + # Keys for config params + VALUE = 'value' + FN = 'fn' # pylint: disable=invalid-name + + ## @} + + + def __init__(self, oOptions): + """ + Initialize internals + """ + self._oOptions = oOptions; + self._sTestBoxHelper = None; + + # Signed-on state + self._cSignOnAttempts = 0; + self._fSignedOn = False; + self._fNeedReSignOn = False; + self._fFirstSignOn = True; + self._idTestBox = None; + self._sTestBoxName = ''; + self._sTestBoxUuid = self.ksNullUuid; # convenience, assigned below. + + # Command processor. + self._oCommand = TestBoxCommand(self); + + # + # Scratch dir setup. Use /var/tmp instead of /tmp because we may need + # many many GBs for some test scenarios and /tmp can be backed by swap + # or be a fast+small disk of some kind, while /var/tmp is normally + # larger, if slower. /var/tmp is generally not cleaned up on reboot, + # /tmp often is, this would break host panic / triple-fault detection. + # + if self._oOptions.sScratchRoot is None: + if utils.getHostOs() in ('win', 'os2', 'haiku', 'dos'): + # We need *lots* of space, so avoid /tmp as it may be a memory + # file system backed by the swap file, or worse. + self._oOptions.sScratchRoot = tempfile.gettempdir(); + else: + self._oOptions.sScratchRoot = '/var/tmp'; + sSubDir = 'testbox'; + try: + sSubDir = '%s-%u' % (sSubDir, os.getuid()); # pylint: disable=no-member + except: + pass; + self._oOptions.sScratchRoot = os.path.join(self._oOptions.sScratchRoot, sSubDir); + + self._sScratchSpill = os.path.join(self._oOptions.sScratchRoot, 'scratch'); + self._sScratchScripts = os.path.join(self._oOptions.sScratchRoot, 'scripts'); + self._sScratchState = os.path.join(self._oOptions.sScratchRoot, 'state'); # persistant storage. + + for sDir in [self._oOptions.sScratchRoot, self._sScratchSpill, self._sScratchScripts, self._sScratchState]: + if not os.path.isdir(sDir): + os.makedirs(sDir, 0o700); + + # We count consecutive reinitScratch failures and will reboot the + # testbox after a while in the hope that it will correct the issue. + self._cReinitScratchErrors = 0; + + # + # Mount builds and test resources if requested. + # + self.mountShares(); + + # + # Sign-on parameters: Packed into list of records of format: + # { <Parameter ID>: { <Current value>, <Check function> } } + # + self._ddSignOnParams = \ + { + constants.tbreq.ALL_PARAM_TESTBOX_UUID: { self.VALUE: self._getHostSystemUuid(), self.FN: None }, + constants.tbreq.SIGNON_PARAM_OS: { self.VALUE: utils.getHostOs(), self.FN: None }, + constants.tbreq.SIGNON_PARAM_OS_VERSION: { self.VALUE: utils.getHostOsVersion(), self.FN: None }, + constants.tbreq.SIGNON_PARAM_CPU_ARCH: { self.VALUE: utils.getHostArch(), self.FN: None }, + constants.tbreq.SIGNON_PARAM_CPU_VENDOR: { self.VALUE: self._getHostCpuVendor(), self.FN: None }, + constants.tbreq.SIGNON_PARAM_CPU_NAME: { self.VALUE: self._getHostCpuName(), self.FN: None }, + constants.tbreq.SIGNON_PARAM_CPU_REVISION: { self.VALUE: self._getHostCpuRevision(), self.FN: None }, + constants.tbreq.SIGNON_PARAM_HAS_HW_VIRT: { self.VALUE: self._hasHostHwVirt(), self.FN: None }, + constants.tbreq.SIGNON_PARAM_HAS_NESTED_PAGING:{ self.VALUE: self._hasHostNestedPaging(), self.FN: None }, + constants.tbreq.SIGNON_PARAM_HAS_64_BIT_GUEST: { self.VALUE: self._can64BitGuest(), self.FN: None }, + constants.tbreq.SIGNON_PARAM_HAS_IOMMU: { self.VALUE: self._hasHostIoMmu(), self.FN: None }, + #constants.tbreq.SIGNON_PARAM_WITH_RAW_MODE: { self.VALUE: self._withRawModeSupport(), self.FN: None }, + constants.tbreq.SIGNON_PARAM_SCRIPT_REV: { self.VALUE: self._getScriptRev(), self.FN: None }, + constants.tbreq.SIGNON_PARAM_REPORT: { self.VALUE: self._getHostReport(), self.FN: None }, + constants.tbreq.SIGNON_PARAM_PYTHON_VERSION: { self.VALUE: self._getPythonHexVersion(), self.FN: None }, + constants.tbreq.SIGNON_PARAM_CPU_COUNT: { self.VALUE: None, self.FN: utils.getPresentCpuCount }, + constants.tbreq.SIGNON_PARAM_MEM_SIZE: { self.VALUE: None, self.FN: self._getHostMemSize }, + constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE: { self.VALUE: None, self.FN: self._getFreeScratchSpace }, + } + for sItem in self._ddSignOnParams: # pylint: disable=consider-using-dict-items + if self._ddSignOnParams[sItem][self.FN] is not None: + self._ddSignOnParams[sItem][self.VALUE] = self._ddSignOnParams[sItem][self.FN]() + + testboxcommons.log('Starting Test Box script (%s)' % (self._getScriptRev(),)); + testboxcommons.log('Test Manager URL: %s' % self._oOptions.sTestManagerUrl,) + testboxcommons.log('Scratch root path: %s' % self._oOptions.sScratchRoot,) + for sItem in self._ddSignOnParams: # pylint: disable=consider-using-dict-items + testboxcommons.log('Sign-On value %18s: %s' % (sItem, self._ddSignOnParams[sItem][self.VALUE])); + + # + # The System UUID is the primary identification of the machine, so + # refuse to cooperate if it's NULL. + # + self._sTestBoxUuid = self.getSignOnParam(constants.tbreq.ALL_PARAM_TESTBOX_UUID); + if self._sTestBoxUuid == self.ksNullUuid: + raise TestBoxScriptException('Couldn\'t determine the System UUID, please use --system-uuid to specify it.'); + + # + # Export environment variables, clearing any we don't know yet. + # + for sEnvVar in self._oOptions.asEnvVars: + iEqual = sEnvVar.find('='); + if iEqual == -1: # No '=', remove it. + if sEnvVar in os.environ: + del os.environ[sEnvVar]; + elif iEqual > 0: # Set it. + os.environ[sEnvVar[:iEqual]] = sEnvVar[iEqual+1:]; + else: # Starts with '=', bad user. + raise TestBoxScriptException('Invalid -E argument: "%s"' % (sEnvVar,)); + + os.environ['TESTBOX_PATH_BUILDS'] = self._oOptions.sBuildsPath; + os.environ['TESTBOX_PATH_RESOURCES'] = self._oOptions.sTestRsrcPath; + os.environ['TESTBOX_PATH_SCRATCH'] = self._sScratchSpill; + os.environ['TESTBOX_PATH_SCRIPTS'] = self._sScratchScripts; + os.environ['TESTBOX_PATH_UPLOAD'] = self._sScratchSpill; ## @todo drop the UPLOAD dir? + os.environ['TESTBOX_HAS_HW_VIRT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_HW_VIRT); + os.environ['TESTBOX_HAS_NESTED_PAGING'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_NESTED_PAGING); + os.environ['TESTBOX_HAS_IOMMU'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_IOMMU); + os.environ['TESTBOX_SCRIPT_REV'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRIPT_REV); + os.environ['TESTBOX_CPU_COUNT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_CPU_COUNT); + os.environ['TESTBOX_MEM_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_MEM_SIZE); + os.environ['TESTBOX_SCRATCH_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE); + #TODO: os.environ['TESTBOX_WITH_RAW_MODE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_WITH_RAW_MODE); + os.environ['TESTBOX_WITH_RAW_MODE'] = str(self._withRawModeSupport()); + os.environ['TESTBOX_MANAGER_URL'] = self._oOptions.sTestManagerUrl; + os.environ['TESTBOX_UUID'] = self._sTestBoxUuid; + os.environ['TESTBOX_REPORTER'] = 'remote'; + os.environ['TESTBOX_NAME'] = ''; + os.environ['TESTBOX_ID'] = ''; + os.environ['TESTBOX_TEST_SET_ID'] = ''; + os.environ['TESTBOX_TIMEOUT'] = '0'; + os.environ['TESTBOX_TIMEOUT_ABS'] = '0'; + + if utils.getHostOs() == 'win': + os.environ['COMSPEC'] = os.path.join(os.environ['SystemRoot'], 'System32', 'cmd.exe'); + # Currently omitting any kBuild tools. + + def mountShares(self): + """ + Mounts the shares. + Raises exception on failure. + """ + self._mountShare(self._oOptions.sBuildsPath, self._oOptions.sBuildsServerType, self._oOptions.sBuildsServerName, + self._oOptions.sBuildsServerShare, + self._oOptions.sBuildsServerUser, self._oOptions.sBuildsServerPasswd, + self._oOptions.sBuildsServerMountOpt, 'builds'); + self._mountShare(self._oOptions.sTestRsrcPath, self._oOptions.sTestRsrcServerType, self._oOptions.sTestRsrcServerName, + self._oOptions.sTestRsrcServerShare, + self._oOptions.sTestRsrcServerUser, self._oOptions.sTestRsrcServerPasswd, + self._oOptions.sTestRsrcServerMountOpt, 'testrsrc'); + return True; + + def _mountShare(self, sMountPoint, sType, sServer, sShare, sUser, sPassword, sMountOpt, sWhat): + """ + Mounts the specified share if needed. + Raises exception on failure. + """ + # Only mount if the type is specified. + if sType is None: + return True; + + # Test if already mounted. + sTestFile = os.path.join(sMountPoint + os.path.sep, os.path.basename(sShare) + '-new.txt'); + if os.path.isfile(sTestFile): + return True; + + # + # Platform specific mount code. + # + sHostOs = utils.getHostOs() + if sHostOs in ('darwin', 'freebsd'): + if sMountOpt != '': + sMountOpt = ',' + sMountOpt + utils.sudoProcessCall(['/sbin/umount', sMountPoint]); + utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]); + utils.sudoProcessCall(['/usr/sbin/chown', str(os.getuid()), sMountPoint]); # pylint: disable=no-member + if sType == 'cifs': + # Note! no smb://server/share stuff here, 10.6.8 didn't like it. + utils.processOutputChecked(['/sbin/mount_smbfs', + '-o', + 'automounted,nostreams,soft,noowners,noatime,rdonly' + sMountOpt, + '-f', '0555', '-d', '0555', + '//%s:%s@%s/%s' % (sUser, sPassword, sServer, sShare), + sMountPoint]); + else: + raise TestBoxScriptException('Unsupported server type %s.' % (sType,)); + + elif sHostOs == 'linux': + if sMountOpt != '': + sMountOpt = ',' + sMountOpt + utils.sudoProcessCall(['/bin/umount', sMountPoint]); + utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]); + if sType == 'cifs': + utils.sudoProcessOutputChecked(['/bin/mount', '-t', 'cifs', + '-o', + 'user=' + sUser + + ',password=' + sPassword + + ',sec=ntlmv2' + + ',uid=' + str(os.getuid()) # pylint: disable=no-member + + ',gid=' + str(os.getgid()) # pylint: disable=no-member + + ',nounix,file_mode=0555,dir_mode=0555,soft,ro' + + sMountOpt, + '//%s/%s' % (sServer, sShare), + sMountPoint]); + elif sType == 'nfs': + utils.sudoProcessOutputChecked(['/bin/mount', '-t', 'nfs', + '-o', 'soft,ro' + sMountOpt, + '%s:%s' % (sServer, sShare if sShare.find('/') >= 0 else ('/export/' + sShare)), + sMountPoint]); + + else: + raise TestBoxScriptException('Unsupported server type %s.' % (sType,)); + + elif sHostOs == 'solaris': + if sMountOpt != '': + sMountOpt = ',' + sMountOpt + utils.sudoProcessCall(['/sbin/umount', sMountPoint]); + utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]); + if sType == 'cifs': + ## @todo This stuff doesn't work on wei01-x4600b.de.oracle.com running 11.1. FIXME! + oPasswdFile = tempfile.TemporaryFile(); # pylint: disable=consider-using-with + oPasswdFile.write(sPassword + '\n'); + oPasswdFile.flush(); + utils.sudoProcessOutputChecked(['/sbin/mount', '-F', 'smbfs', + '-o', + 'user=' + sUser + + ',uid=' + str(os.getuid()) # pylint: disable=no-member + + ',gid=' + str(os.getgid()) # pylint: disable=no-member + + ',fileperms=0555,dirperms=0555,noxattr,ro' + + sMountOpt, + '//%s/%s' % (sServer, sShare), + sMountPoint], + stdin = oPasswdFile); + oPasswdFile.close(); + elif sType == 'nfs': + utils.sudoProcessOutputChecked(['/sbin/mount', '-F', 'nfs', + '-o', 'noxattr,ro' + sMountOpt, + '%s:%s' % (sServer, sShare if sShare.find('/') >= 0 else ('/export/' + sShare)), + sMountPoint]); + + else: + raise TestBoxScriptException('Unsupported server type %s.' % (sType,)); + + + elif sHostOs == 'win': + if sType != 'cifs': + raise TestBoxScriptException('Only CIFS mounts are supported on Windows.'); + utils.processCall(['net', 'use', sMountPoint, '/d']); + utils.processOutputChecked(['net', 'use', sMountPoint, + '\\\\' + sServer + '\\' + sShare, + sPassword, + '/USER:' + sUser,]); + else: + raise TestBoxScriptException('Unsupported host %s' % (sHostOs,)); + + # + # Re-test. + # + if not os.path.isfile(sTestFile): + raise TestBoxException('Failed to mount %s (%s[%s]) at %s: %s not found' + % (sWhat, sServer, sShare, sMountPoint, sTestFile)); + + return True; + + ## @name Signon property releated methods. + # @{ + + def _getHelperOutput(self, sCmd): + """ + Invokes TestBoxHelper to obtain information hard to access from python. + """ + if self._sTestBoxHelper is None: + if not utils.isRunningFromCheckout(): + # See VBoxTestBoxScript.zip for layout. + self._sTestBoxHelper = os.path.join(g_ksValidationKitDir, utils.getHostOs(), utils.getHostArch(), \ + 'TestBoxHelper'); + else: # Only for in-tree testing, so don't bother be too accurate right now. + sType = os.environ.get('KBUILD_TYPE', 'debug'); + self._sTestBoxHelper = os.path.join(g_ksValidationKitDir, os.pardir, os.pardir, os.pardir, 'out', \ + utils.getHostOsDotArch(), sType, 'testboxscript', \ + utils.getHostOs(), utils.getHostArch(), \ + 'TestBoxHelper'); + if utils.getHostOs() in ['win', 'os2']: + self._sTestBoxHelper += '.exe'; + + return utils.processOutputChecked([self._sTestBoxHelper, sCmd]).strip(); + + def _getHelperOutputTristate(self, sCmd, fDunnoValue): + """ + Invokes TestBoxHelper to obtain information hard to access from python. + """ + sValue = self._getHelperOutput(sCmd); + sValue = sValue.lower(); + if sValue == 'true': + return True; + if sValue == 'false': + return False; + if sValue not in ('dunno', 'none',): + raise TestBoxException('Unexpected response "%s" to helper command "%s"' % (sValue, sCmd)); + return fDunnoValue; + + + @staticmethod + def _isUuidGood(sUuid): + """ + Checks if the UUID looks good. + + There are systems with really bad UUIDs, for instance + "03000200-0400-0500-0006-000700080009". + """ + if sUuid == TestBoxScript.ksNullUuid: + return False; + sUuid = sUuid.lower(); + for sDigit in ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']: + if sUuid.count(sDigit) > 16: + return False; + return True; + + def _getHostSystemUuid(self): + """ + Get the system UUID string from the System, return null-uuid if + unable to get retrieve it. + """ + if self._oOptions.sSystemUuid is not None: + return self._oOptions.sSystemUuid; + + sUuid = self.ksNullUuid; + + # + # Try get at the firmware UUID. + # + if utils.getHostOs() == 'linux': + # NOTE: This requires to have kernel option enabled: + # Firmware Drivers -> Export DMI identification via sysfs to userspace + if os.path.exists('/sys/devices/virtual/dmi/id/product_uuid'): + try: + sVar = utils.sudoProcessOutputChecked(['cat', '/sys/devices/virtual/dmi/id/product_uuid']); + sUuid = str(uuid.UUID(sVar.strip())); + except: + pass; + ## @todo consider dmidecoder? What about EFI systems? + + elif utils.getHostOs() == 'win': + # Windows: WMI + try: + import win32com.client; # pylint: disable=import-error + oWmi = win32com.client.Dispatch('WbemScripting.SWbemLocator'); + oWebm = oWmi.ConnectServer('.', 'root\\cimv2'); + for oItem in oWebm.ExecQuery('SELECT * FROM Win32_ComputerSystemProduct'): + if oItem.UUID is not None: + sUuid = str(uuid.UUID(oItem.UUID)); + except: + pass; + + elif utils.getHostOs() == 'darwin': + try: + sVar = utils.processOutputChecked(['/bin/sh', '-c', + '/usr/sbin/ioreg -k IOPlatformUUID' \ + + '| /usr/bin/grep IOPlatformUUID' \ + + '| /usr/bin/head -1']); + sVar = sVar.strip()[-(len(self.ksNullUuid) + 1):-1]; + sUuid = str(uuid.UUID(sVar)); + except: + pass; + + elif utils.getHostOs() == 'solaris': + # Solaris: The smbios util. + try: + sVar = utils.processOutputChecked(['/bin/sh', '-c', + '/usr/sbin/smbios ' \ + + '| /usr/xpg4/bin/sed -ne \'s/^.*UUID: *//p\'' \ + + '| /usr/bin/head -1']); + sUuid = str(uuid.UUID(sVar.strip())); + except: + pass; + + if self._isUuidGood(sUuid): + return sUuid; + + # + # Try add the MAC address. + # uuid.getnode may provide it, or it may return a random number... + # + lMacAddr = uuid.getnode(); + sNode = '%012x' % (lMacAddr,) + if lMacAddr == uuid.getnode() and lMacAddr != 0 and len(sNode) == 12: + return sUuid[:-12] + sNode; + + return sUuid; + + def _getHostCpuVendor(self): + """ + Get the CPUID vendor string on intel HW. + """ + return self._getHelperOutput('cpuvendor'); + + def _getHostCpuName(self): + """ + Get the CPU name/description string. + """ + return self._getHelperOutput('cpuname'); + + def _getHostCpuRevision(self): + """ + Get the CPU revision (family/model/stepping) value. + """ + return self._getHelperOutput('cpurevision'); + + def _hasHostHwVirt(self): + """ + Check if the host supports AMD-V or VT-x + """ + if self._oOptions.fHasHwVirt is None: + self._oOptions.fHasHwVirt = self._getHelperOutput('cpuhwvirt'); + return self._oOptions.fHasHwVirt; + + def _hasHostNestedPaging(self): + """ + Check if the host supports nested paging. + """ + if not self._hasHostHwVirt(): + return False; + if self._oOptions.fHasNestedPaging is None: + self._oOptions.fHasNestedPaging = self._getHelperOutputTristate('nestedpaging', False); + return self._oOptions.fHasNestedPaging; + + def _can64BitGuest(self): + """ + Check if the we (VBox) can run 64-bit guests. + """ + if not self._hasHostHwVirt(): + return False; + if self._oOptions.fCan64BitGuest is None: + self._oOptions.fCan64BitGuest = self._getHelperOutputTristate('longmode', True); + return self._oOptions.fCan64BitGuest; + + def _hasHostIoMmu(self): + """ + Check if the host has an I/O MMU of the VT-d kind. + """ + if not self._hasHostHwVirt(): + return False; + if self._oOptions.fHasIoMmu is None: + ## @todo Any way to figure this one out on any host OS? + self._oOptions.fHasIoMmu = False; + return self._oOptions.fHasIoMmu; + + def _withRawModeSupport(self): + """ + Check if the testbox is configured with raw-mode support or not. + """ + if self._oOptions.fWithRawMode is None: + self._oOptions.fWithRawMode = True; + return self._oOptions.fWithRawMode; + + def _getHostReport(self): + """ + Generate a report about the host hardware and software. + """ + return self._getHelperOutput('report'); + + + def _getHostMemSize(self): + """ + Gets the amount of physical memory on the host (and accessible to the + OS, i.e. don't report stuff over 4GB if Windows doesn't wanna use it). + Unit: MiB. + """ + cMbMemory = long(self._getHelperOutput('memsize').strip()) / (1024 * 1024); + + # Round it. + cMbMemory = long(math.floor(cMbMemory / self.kcMbMemoryRounding)) * self.kcMbMemoryRounding; + return cMbMemory; + + def _getFreeScratchSpace(self): + """ + Get free space on the volume where scratch directory is located and + return it in bytes rounded down to nearest 64MB + (currently works on Linux only) + Unit: MiB. + """ + if platform.system() == 'Windows': + import ctypes + cTypeMbFreeSpace = ctypes.c_ulonglong(0) + ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(self._oOptions.sScratchRoot), None, None, + ctypes.pointer(cTypeMbFreeSpace)) + cMbFreeSpace = cTypeMbFreeSpace.value + else: + stats = os.statvfs(self._oOptions.sScratchRoot); # pylint: disable=no-member + cMbFreeSpace = stats.f_frsize * stats.f_bfree + + # Convert to MB + cMbFreeSpace = long(cMbFreeSpace) /(1024 * 1024) + + # Round free space size + cMbFreeSpace = long(math.floor(cMbFreeSpace / self.kcMbScratchSpaceRounding)) * self.kcMbScratchSpaceRounding; + return cMbFreeSpace; + + def _getScriptRev(self): + """ + The script (subversion) revision number. + """ + sRev = '@VBOX_SVN_REV@'; + sRev = sRev.strip(); # just in case... + try: + _ = int(sRev); + except: + return __version__[11:-1].strip(); + return sRev; + + def _getPythonHexVersion(self): + """ + The python hex version number. + """ + uHexVersion = getattr(sys, 'hexversion', None); + if uHexVersion is None: + uHexVersion = (sys.version_info[0] << 24) | (sys.version_info[1] << 16) | (sys.version_info[2] << 8); + if sys.version_info[3] == 'final': + uHexVersion |= 0xf0; + return uHexVersion; + + # @} + + def openTestManagerConnection(self): + """ + Opens up a connection to the test manager. + + Raises exception on failure. + """ + return TestBoxConnection(self._oOptions.sTestManagerUrl, self._idTestBox, self._sTestBoxUuid); + + def getSignOnParam(self, sName): + """ + Returns a sign-on parameter value as string. + Raises exception if the name is incorrect. + """ + return str(self._ddSignOnParams[sName][self.VALUE]); + + def getPathState(self): + """ + Get the path to the state dir in the scratch area. + """ + return self._sScratchState; + + def getPathScripts(self): + """ + Get the path to the scripts dir (TESTBOX_PATH_SCRIPTS) in the scratch area. + """ + return self._sScratchScripts; + + def getPathSpill(self): + """ + Get the path to the spill dir (TESTBOX_PATH_SCRATCH) in the scratch area. + """ + return self._sScratchSpill; + + def getPathBuilds(self): + """ + Get the path to the builds. + """ + return self._oOptions.sBuildsPath; + + def getTestBoxId(self): + """ + Get the TestBox ID for state saving purposes. + """ + return self._idTestBox; + + def getTestBoxName(self): + """ + Get the TestBox name for state saving purposes. + """ + return self._sTestBoxName; + + def _reinitScratch(self, fnLog, fUseTheForce): + """ + Wipes the scratch directories and re-initializes them. + + No exceptions raise, returns success indicator instead. + """ + if fUseTheForce is None: + fUseTheForce = self._fFirstSignOn; + + class ErrorCallback(object): # pylint: disable=too-few-public-methods + """ + Callbacks + state for the cleanup. + """ + def __init__(self): + self.fRc = True; + def onErrorCallback(self, sFnName, sPath, aXcptInfo): + """ Logs error during shutil.rmtree operation. """ + fnLog('Error removing "%s": fn=%s %s' % (sPath, sFnName, aXcptInfo[1])); + self.fRc = False; + oRc = ErrorCallback(); + + # + # Cleanup. + # + for sName in os.listdir(self._oOptions.sScratchRoot): + sFullName = os.path.join(self._oOptions.sScratchRoot, sName); + try: + if os.path.isdir(sFullName): + shutil.rmtree(sFullName, False, oRc.onErrorCallback); + else: + os.remove(sFullName); + if os.path.exists(sFullName): + raise Exception('Still exists after deletion, weird.'); + except Exception as oXcpt: + if fUseTheForce is True \ + and utils.getHostOs() not in ['win', 'os2'] \ + and len(sFullName) >= 8 \ + and sFullName[0] == '/' \ + and sFullName[1] != '/' \ + and sFullName.find('/../') < 0: + fnLog('Problems deleting "%s" (%s) using the force...' % (sFullName, oXcpt)); + try: + if os.path.isdir(sFullName): + iRc = utils.sudoProcessCall(['/bin/rm', '-Rf', sFullName]) + else: + iRc = utils.sudoProcessCall(['/bin/rm', '-f', sFullName]) + if iRc != 0: + raise Exception('exit code %s' % iRc); + if os.path.exists(sFullName): + raise Exception('Still exists after forced deletion, weird^2.'); + except: + fnLog('Error sudo deleting "%s": %s' % (sFullName, oXcpt)); + oRc.fRc = False; + else: + fnLog('Error deleting "%s": %s' % (sFullName, oXcpt)); + oRc.fRc = False; + + # Display files left behind. + def dirEnumCallback(sName, oStat): + """ callback for dirEnumerateTree """ + fnLog(u'%s %s' % (utils.formatFileStat(oStat) if oStat is not None else '????????????', sName)); + utils.dirEnumerateTree(self._oOptions.sScratchRoot, dirEnumCallback); + + # + # Re-create the directories. + # + for sDir in [self._oOptions.sScratchRoot, self._sScratchSpill, self._sScratchScripts, self._sScratchState]: + if not os.path.isdir(sDir): + try: + os.makedirs(sDir, 0o700); + except Exception as oXcpt: + fnLog('Error creating "%s": %s' % (sDir, oXcpt)); + oRc.fRc = False; + + if oRc.fRc is True: + self._cReinitScratchErrors = 0; + else: + self._cReinitScratchErrors += 1; + return oRc.fRc; + + def reinitScratch(self, fnLog = testboxcommons.log, fUseTheForce = None, cRetries = 0, cMsDelay = 5000): + """ + Wipes the scratch directories and re-initializes them. + + Will retry according to the cRetries and cMsDelay parameters. Windows + forces us to apply this hack as it ships with services asynchronously + scanning files after they execute, thus racing us cleaning up after a + test. On testboxwin3 we had frequent trouble with aelupsvc.dll keeping + vts_rm.exe kind of open, somehow preventing us from removing the + directory containing it, despite not issuing any errors deleting the + file itself. The service is called "Application Experience", which + feels like a weird joke here. + + No exceptions raise, returns success indicator instead. + """ + fRc = self._reinitScratch(fnLog, fUseTheForce) + while fRc is False and cRetries > 0: + time.sleep(cMsDelay / 1000.0); + fnLog('reinitScratch: Retrying...'); + fRc = self._reinitScratch(fnLog, fUseTheForce) + cRetries -= 1; + return fRc; + + + def _doSignOn(self): + """ + Worker for _maybeSignOn that does the actual signing on. + """ + assert not self._oCommand.isRunning(); + + # Reset the siged-on state. + testboxcommons.log('Signing-on...') + self._fSignedOn = False + self._idTestBox = None + self._cSignOnAttempts += 1; + + # Assemble SIGN-ON request parameters and send the request. + dParams = {}; + for sParam in self._ddSignOnParams: # pylint: disable=consider-using-dict-items + dParams[sParam] = self._ddSignOnParams[sParam][self.VALUE]; + oResponse = TestBoxConnection.sendSignOn(self._oOptions.sTestManagerUrl, dParams); + + # Check response. + try: + sResult = oResponse.getStringChecked(constants.tbresp.ALL_PARAM_RESULT); + if sResult != constants.tbresp.STATUS_ACK: + raise TestBoxException('Result is %s' % (sResult,)); + oResponse.checkParameterCount(3); + idTestBox = oResponse.getIntChecked(constants.tbresp.SIGNON_PARAM_ID, 1, 0x7ffffffe); + sTestBoxName = oResponse.getStringChecked(constants.tbresp.SIGNON_PARAM_NAME); + except TestBoxException as err: + testboxcommons.log('Failed to sign-on: %s' % (str(err),)) + testboxcommons.log('Server response: %s' % (oResponse.toString(),)); + return False; + + # Successfully signed on, update the state. + self._fSignedOn = True; + self._fNeedReSignOn = False; + self._cSignOnAttempts = 0; + self._idTestBox = idTestBox; + self._sTestBoxName = sTestBoxName; + + # Update the environment. + os.environ['TESTBOX_ID'] = str(self._idTestBox); + os.environ['TESTBOX_NAME'] = sTestBoxName; + os.environ['TESTBOX_CPU_COUNT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_CPU_COUNT); + os.environ['TESTBOX_MEM_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_MEM_SIZE); + os.environ['TESTBOX_SCRATCH_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE); + + testboxcommons.log('Successfully signed-on with Test Box ID #%s and given the name "%s"' \ + % (self._idTestBox, self._sTestBoxName)); + + # Set up the scratch area. + self.reinitScratch(fUseTheForce = self._fFirstSignOn, cRetries = 2); + + self._fFirstSignOn = False; + return True; + + def _maybeSignOn(self): + """ + Check if Test Box parameters were changed + and do sign-in in case of positive result + """ + + # Skip sign-on check if background command is currently in + # running state (avoid infinite signing on). + if self._oCommand.isRunning(): + return None; + + # Refresh sign-on parameters, changes triggers sign-on. + fNeedSignOn = not self._fSignedOn or self._fNeedReSignOn; + for sItem in self._ddSignOnParams: # pylint: disable=consider-using-dict-items + if self._ddSignOnParams[sItem][self.FN] is None: + continue + + sOldValue = self._ddSignOnParams[sItem][self.VALUE] + self._ddSignOnParams[sItem][self.VALUE] = self._ddSignOnParams[sItem][self.FN]() + if sOldValue != self._ddSignOnParams[sItem][self.VALUE]: + fNeedSignOn = True + testboxcommons.log('Detected %s parameter change: %s -> %s' + % (sItem, sOldValue, self._ddSignOnParams[sItem][self.VALUE],)) + + if fNeedSignOn: + self._doSignOn(); + return None; + + def dispatch(self): + """ + Receive orders from Test Manager and execute them + """ + + (self._idTestBox, self._sTestBoxName, self._fSignedOn) = self._oCommand.resumeIncompleteCommand(); + self._fNeedReSignOn = self._fSignedOn; + if self._fSignedOn: + os.environ['TESTBOX_ID'] = str(self._idTestBox); + os.environ['TESTBOX_NAME'] = self._sTestBoxName; + + while True: + # Make sure we're signed on before trying to do anything. + self._maybeSignOn(); + while not self._fSignedOn: + iFactor = 1 if self._cSignOnAttempts < 100 else 4; + time.sleep(random.randint(self.kcSecMinSignOnDelay * iFactor, self.kcSecMaxSignOnDelay * iFactor)); + self._maybeSignOn(); + + # Retrieve and handle command from the TM. + (oResponse, oConnection) = TestBoxConnection.requestCommandWithConnection(self._oOptions.sTestManagerUrl, + self._idTestBox, + self._sTestBoxUuid, + self._oCommand.isRunning()); + if oResponse is not None: + self._oCommand.handleCommand(oResponse, oConnection); + if oConnection is not None: + if oConnection.isConnected(): + self._oCommand.flushLogOnConnection(oConnection); + oConnection.close(); + + # Automatically reboot if scratch init fails. + #if self._cReinitScratchErrors > 8 and self.reinitScratch(cRetries = 3) is False: + # testboxcommons.log('Scratch does not initialize cleanly after %d attempts, rebooting...' + # % ( self._cReinitScratchErrors, )); + # self._oCommand.doReboot(); + + # delay a wee bit before looping. + ## @todo We shouldn't bother the server too frequently. We should try combine the test reporting done elsewhere + # with the command retrieval done here. I believe tinderclient.pl is capable of doing that. + iFactor = 1; + if self._cReinitScratchErrors > 0: + iFactor = 4; + time.sleep(random.randint(self.kcSecMinDelay * iFactor, self.kcSecMaxDelay * iFactor)); + + # Not reached. + + + @staticmethod + def main(): + """ + Main function a la C/C++. Returns exit code. + """ + + # + # Parse arguments. + # + sDefShareType = 'nfs' if utils.getHostOs() == 'solaris' else 'cifs'; + if utils.getHostOs() in ('win', 'os2'): + sDefTestRsrc = 'T:'; + sDefBuilds = 'U:'; + elif utils.getHostOs() == 'darwin': + sDefTestRsrc = '/Volumes/testrsrc'; + sDefBuilds = '/Volumes/builds'; + else: + sDefTestRsrc = '/mnt/testrsrc'; + sDefBuilds = '/mnt/builds'; + + class MyOptionParser(OptionParser): + """ We need to override the exit code on --help, error and so on. """ + def __init__(self, *args, **kwargs): + OptionParser.__init__(self, *args, **kwargs); + def exit(self, status = 0, msg = None): + OptionParser.exit(self, TBS_EXITCODE_SYNTAX, msg); + + parser = MyOptionParser(version=__version__[11:-1].strip()); + for sMixed, sDefault, sDesc in [('Builds', sDefBuilds, 'builds'), ('TestRsrc', sDefTestRsrc, 'test resources') ]: + sLower = sMixed.lower(); + sPrefix = 's' + sMixed; + parser.add_option('--' + sLower + '-path', + dest=sPrefix + 'Path', metavar='<abs-path>', default=sDefault, + help='Where ' + sDesc + ' can be found'); + parser.add_option('--' + sLower + '-server-type', + dest=sPrefix + 'ServerType', metavar='<nfs|cifs>', default=sDefShareType, + help='The type of server, cifs (default) or nfs. If empty, we won\'t try mount anything.'); + parser.add_option('--' + sLower + '-server-name', + dest=sPrefix + 'ServerName', metavar='<server>', + default='vboxstor.de.oracle.com' if sLower == 'builds' else 'teststor.de.oracle.com', + help='The name of the server with the builds.'); + parser.add_option('--' + sLower + '-server-share', + dest=sPrefix + 'ServerShare', metavar='<share>', default=sLower, + help='The name of the builds share.'); + parser.add_option('--' + sLower + '-server-user', + dest=sPrefix + 'ServerUser', metavar='<user>', default='guestr', + help='The user name to use when accessing the ' + sDesc + ' share.'); + parser.add_option('--' + sLower + '-server-passwd', '--' + sLower + '-server-password', + dest=sPrefix + 'ServerPasswd', metavar='<password>', default='guestr', + help='The password to use when accessing the ' + sDesc + ' share.'); + parser.add_option('--' + sLower + '-server-mountopt', + dest=sPrefix + 'ServerMountOpt', metavar='<mountopt>', default='', + help='The mount options to use when accessing the ' + sDesc + ' share.'); + + parser.add_option("--test-manager", metavar="<url>", + dest="sTestManagerUrl", + help="Test Manager URL", + default="http://tindertux.de.oracle.com/testmanager") + parser.add_option("--scratch-root", metavar="<abs-path>", + dest="sScratchRoot", + help="Path to the scratch directory", + default=None) + parser.add_option("--system-uuid", metavar="<uuid>", + dest="sSystemUuid", + help="The system UUID of the testbox, used for uniquely identifiying the machine", + default=None) + parser.add_option("--hwvirt", + dest="fHasHwVirt", action="store_true", default=None, + help="Hardware virtualization available in the CPU"); + parser.add_option("--no-hwvirt", + dest="fHasHwVirt", action="store_false", default=None, + help="Hardware virtualization not available in the CPU"); + parser.add_option("--nested-paging", + dest="fHasNestedPaging", action="store_true", default=None, + help="Nested paging is available"); + parser.add_option("--no-nested-paging", + dest="fHasNestedPaging", action="store_false", default=None, + help="Nested paging is not available"); + parser.add_option("--64-bit-guest", + dest="fCan64BitGuest", action="store_true", default=None, + help="Host can execute 64-bit guests"); + parser.add_option("--no-64-bit-guest", + dest="fCan64BitGuest", action="store_false", default=None, + help="Host cannot execute 64-bit guests"); + parser.add_option("--io-mmu", + dest="fHasIoMmu", action="store_true", default=None, + help="I/O MMU available"); + parser.add_option("--no-io-mmu", + dest="fHasIoMmu", action="store_false", default=None, + help="No I/O MMU available"); + parser.add_option("--raw-mode", + dest="fWithRawMode", action="store_true", default=None, + help="Use raw-mode on this host."); + parser.add_option("--no-raw-mode", + dest="fWithRawMode", action="store_false", default=None, + help="Disables raw-mode tests on this host."); + parser.add_option("--pidfile", + dest="sPidFile", default=None, + help="For the parent script, ignored."); + parser.add_option("-E", "--putenv", metavar = "<variable>=<value>", action = "append", + dest = "asEnvVars", default = [], + help = "Sets an environment variable. Can be repeated."); + def sbp_callback(option, opt_str, value, parser): + _, _, _ = opt_str, value, option + parser.values.sTestManagerUrl = 'http://10.162.100.8/testmanager/' + parser.values.sBuildsServerName = 'vbox-st02.ru.oracle.com' + parser.values.sTestRsrcServerName = 'vbox-st02.ru.oracle.com' + parser.values.sTestRsrcServerShare = 'scratch/data/testrsrc' + parser.add_option("--spb", "--load-sbp-defaults", action="callback", callback=sbp_callback, + help="Load defaults for the sbp setup.") + + (oOptions, args) = parser.parse_args() + # Check command line + if args != []: + parser.print_help(); + return TBS_EXITCODE_SYNTAX; + + if oOptions.sSystemUuid is not None: + uuid.UUID(oOptions.sSystemUuid); + if not oOptions.sTestManagerUrl.startswith('http://') \ + and not oOptions.sTestManagerUrl.startswith('https://'): + print('Syntax error: Invalid test manager URL "%s"' % (oOptions.sTestManagerUrl,)); + return TBS_EXITCODE_SYNTAX; + + for sPrefix in ['sBuilds', 'sTestRsrc']: + sType = getattr(oOptions, sPrefix + 'ServerType'); + if sType is None or not sType.strip(): + setattr(oOptions, sPrefix + 'ServerType', None); + elif sType not in ['cifs', 'nfs']: + print('Syntax error: Invalid server type "%s"' % (sType,)); + return TBS_EXITCODE_SYNTAX; + + + # + # Instantiate the testbox script and start dispatching work. + # + try: + oTestBoxScript = TestBoxScript(oOptions); + except TestBoxScriptException as oXcpt: + print('Error: %s' % (oXcpt,)); + return TBS_EXITCODE_SYNTAX; + oTestBoxScript.dispatch(); + + # Not supposed to get here... + return TBS_EXITCODE_FAILURE; + + + +if __name__ == '__main__': + sys.exit(TestBoxScript.main()); + |