diff options
Diffstat (limited to 'src/VBox/ValidationKit/testboxscript/testboxtasks.py')
| -rwxr-xr-x | src/VBox/ValidationKit/testboxscript/testboxtasks.py | 944 |
1 files changed, 944 insertions, 0 deletions
diff --git a/src/VBox/ValidationKit/testboxscript/testboxtasks.py b/src/VBox/ValidationKit/testboxscript/testboxtasks.py new file mode 100755 index 00000000..4d34cbd8 --- /dev/null +++ b/src/VBox/ValidationKit/testboxscript/testboxtasks.py @@ -0,0 +1,944 @@ +# -*- coding: utf-8 -*- +# $Id: testboxtasks.py $ + +""" +TestBox Script - Async Tasks. +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Standard python imports. +from datetime import datetime +import os +import re +import signal; +import sys +import subprocess +import threading +import time + +# Validation Kit imports. +from common import constants +from common import utils; +from common import webutils; +import testboxcommons + +# Figure where we are. +try: __file__ +except: __file__ = sys.argv[0]; +g_ksTestScriptDir = os.path.dirname(os.path.abspath(__file__)); + + + +class TestBoxBaseTask(object): + """ + Asynchronous task employing a thread to do the actual work. + """ + + ## Time to wait for a task to terminate. + kcSecTerminateTimeout = 60 + + def __init__(self, oTestBoxScript, cSecTimeout, fnThreadProc): + self._oTestBoxScript = oTestBoxScript; + self._cSecTimeout = cSecTimeout; + self._tsSecStarted = utils.timestampSecond(); + self.__oRLock = threading.RLock(); + self._oCv = threading.Condition(self.__oRLock); + self._fRunning = True; # Protected by lock. + self._fShouldTerminate = False; # Protected by lock. + + # Spawn the worker thread. + self._oThread = threading.Thread(target=fnThreadProc); + self._oThread.daemon = True; + self._oThread.start(); + + def _lock(self): + """ Take the CV lock. """ + self._oCv.acquire(); + + def _unlock(self): + """ Release the CV lock. """ + self._oCv.release(); + + def _complete(self): + """ + Indicate that the task is complete, waking up the main thread. + Usually called at the end of the thread procedure. + """ + self._lock(); + self._fRunning = False; + self._oCv.notifyAll(); # pylint: disable=deprecated-method + self._unlock(); + + def isRunning(self): + """ Check if the task is still running. """ + self._lock(); + fRunning = self._fRunning; + self._unlock(); + return fRunning; + + def wait(self, cSecTimeout): + """ Wait for the task to complete. """ + self._lock(); + fRunning = self._fRunning; + if fRunning is True and cSecTimeout > 0: + self._oCv.wait(cSecTimeout) + self._unlock(); + return fRunning; + + def terminate(self, cSecTimeout = kcSecTerminateTimeout): + """ Terminate the task. """ + self._lock(); + self._fShouldTerminate = True; + self._unlock(); + + return self.wait(cSecTimeout); + + def _shouldTerminate(self): + """ + Returns True if we should terminate, False if not. + """ + self._lock(); + fShouldTerminate = self._fShouldTerminate is True; + self._unlock(); + return fShouldTerminate; + + +class TestBoxTestDriverTask(TestBoxBaseTask): + """ + Base class for tasks involving test drivers. + """ + + ## When to flush the backlog of log messages. + kcchMaxBackLog = 32768; + + ## The backlog sync time (seconds). + kcSecBackLogFlush = 30; + + ## The timeout for the cleanup job (5 mins). + kcSecCleanupTimeout = 300; + ## The timeout to wait for the abort command before killing it. + kcSecAbortTimeout = 300; + + ## The timeout to wait for the final output to be processed. + kcSecFinalOutputTimeout = 180; + ## The timeout to wait for the abort command output to be processed. + kcSecAbortCmdOutputTimeout = 30; + ## The timeout to wait for the terminate output to be processed. + kcSecTerminateOutputTimeout = 30; + ## The timeout to wait for the kill output to be processed. + kcSecKillOutputTimeout = 30; + + ## The timeout for talking to the test manager. + ksecTestManagerTimeout = 60; + + + def __init__(self, oTestBoxScript, fnThreadProc, cSecTimeout, idResult, sScriptCmdLine): + """ + Class instance init + """ + # Init our instance data. + self._idResult = idResult; + self._sScriptCmdLine = sScriptCmdLine; + self._oChild = None; + self._oBackLogLock = threading.RLock(); + self._oBackLogFlushLock = threading.RLock(); + self._asBackLog = []; + self._cchBackLog = 0; + self._secTsBackLogFlush = utils.timestampSecond(); + + # Init super. + TestBoxBaseTask.__init__(self, oTestBoxScript, cSecTimeout, fnThreadProc); + + def terminate(self, cSecTimeout = kcSecCleanupTimeout): + """ Reimplement with higher default timeout. """ + return TestBoxBaseTask.terminate(self, cSecTimeout); + + def _logFlush(self, oGivenConnection = None): + """ + Flushes the log to the test manager. + + No exceptions. + """ + fRc = True; + + with self._oBackLogFlushLock: + # Grab the current back log. + with self._oBackLogLock: + asBackLog = self._asBackLog; + self._asBackLog = []; + self._cchBackLog = 0; + self._secTsBackLogFlush = utils.timestampSecond(); + + # If there is anything to flush, flush it. + if asBackLog: + sBody = ''; + for sLine in asBackLog: + sBody += sLine + '\n'; + + oConnection = None; + try: + if oGivenConnection is None: + oConnection = self._oTestBoxScript.openTestManagerConnection(); + oConnection.postRequest(constants.tbreq.LOG_MAIN, {constants.tbreq.LOG_PARAM_BODY: sBody}); + oConnection.close(); + else: + oGivenConnection.postRequest(constants.tbreq.LOG_MAIN, {constants.tbreq.LOG_PARAM_BODY: sBody}); + except Exception as oXcpt: + testboxcommons.log('_logFlush error: %s' % (oXcpt,)); + if len(sBody) < self.kcchMaxBackLog * 4: + with self._oBackLogLock: + asBackLog.extend(self._asBackLog); + self._asBackLog = asBackLog; + # Don't restore _cchBackLog as there is no point in retrying immediately. + if oConnection is not None: # Be kind to apache. + try: oConnection.close(); + except: pass; + fRc = False; + + return fRc; + + def flushLogOnConnection(self, oConnection): + """ + Attempts to flush the logon the given connection. + + No exceptions. + """ + return self._logFlush(oConnection); + + def _logInternal(self, sMessage, fPrefix = True, fFlushCheck = False): + """ + Internal logging. + Won't flush the backlog, returns a flush indicator so the caller can + do it instead. + """ + if fPrefix: + try: + oNow = datetime.utcnow(); + sTs = '%02u:%02u:%02u.%06u ' % (oNow.hour, oNow.minute, oNow.second, oNow.microsecond); + except Exception as oXcpt: + sTs = 'oXcpt=%s ' % (oXcpt); + sFullMsg = sTs + sMessage; + else: + sFullMsg = sMessage; + + with self._oBackLogLock: + self._asBackLog.append(sFullMsg); + cchBackLog = self._cchBackLog + len(sFullMsg) + 1; + self._cchBackLog = cchBackLog; + secTsBackLogFlush = self._secTsBackLogFlush; + + testboxcommons.log(sFullMsg); + return fFlushCheck \ + and ( cchBackLog >= self.kcchMaxBackLog \ + or utils.timestampSecond() - secTsBackLogFlush >= self.kcSecBackLogFlush); + + def _log(self, sMessage): + """ + General logging function, will flush. + """ + if self._logInternal(sMessage, fFlushCheck = True): + self._logFlush(); + return True; + + def _reportDone(self, sResult): + """ + Report EXEC job done to the test manager. + + sResult is a value from constants.result. + """ + ## @todo optimize this to use one server connection. + + # + # Log it. + # + assert sResult in constants.result.g_kasValidResults; + self._log('Done %s' % (sResult,)); + + # + # Report it. + # + fRc = True; + secStart = utils.timestampSecond(); + while True: + self._logFlush(); ## @todo Combine this with EXEC_COMPLETED. + oConnection = None; + try: + oConnection = self._oTestBoxScript.openTestManagerConnection(); + oConnection.postRequest(constants.tbreq.EXEC_COMPLETED, {constants.tbreq.EXEC_COMPLETED_PARAM_RESULT: sResult}); + oConnection.close(); + except Exception as oXcpt: + if utils.timestampSecond() - secStart < self.ksecTestManagerTimeout: + self._log('_reportDone exception (%s) - retrying...' % (oXcpt,)); + time.sleep(2); + continue; + self._log('_reportDone error: %s' % (oXcpt,)); + if oConnection is not None: # Be kind to apache. + try: oConnection.close(); + except: pass; + fRc = False; + break; + + # + # Mark the task as completed. + # + self._complete(); + return fRc; + + def _assembleArguments(self, sAction, fWithInterpreter = True): + """ + Creates an argument array for subprocess.Popen, splitting the + sScriptCmdLine like bourne shell would. + fWithInterpreter is used (False) when checking that the script exists. + + Returns None on bad input. + """ + + # + # This is a good place to export the test set id to the environment. + # + os.environ['TESTBOX_TEST_SET_ID'] = str(self._idResult); + cTimeoutLeft = utils.timestampSecond() - self._tsSecStarted; + cTimeoutLeft = 0 if cTimeoutLeft >= self._cSecTimeout else self._cSecTimeout - cTimeoutLeft; + os.environ['TESTBOX_TIMEOUT'] = str(cTimeoutLeft); + os.environ['TESTBOX_TIMEOUT_ABS'] = str(self._tsSecStarted + self._cSecTimeout); + + # + # Do replacements and split the command line into arguments. + # + if self._sScriptCmdLine.find('@ACTION@') >= 0: + sCmdLine = self._sScriptCmdLine.replace('@ACTION@', sAction); + else: + sCmdLine = self._sScriptCmdLine + ' ' + sAction; + for sVar in [ 'TESTBOX_PATH_BUILDS', 'TESTBOX_PATH_RESOURCES', 'TESTBOX_PATH_SCRATCH', 'TESTBOX_PATH_SCRIPTS', + 'TESTBOX_PATH_UPLOAD', 'TESTBOX_UUID', 'TESTBOX_REPORTER', 'TESTBOX_ID', 'TESTBOX_TEST_SET_ID', + 'TESTBOX_TIMEOUT', 'TESTBOX_TIMEOUT_ABS' ]: + if sCmdLine.find('${' + sVar + '}') >= 0: + sCmdLine = sCmdLine.replace('${' + sVar + '}', os.environ[sVar]); + + asArgs = utils.argsSplit(sCmdLine); + + # + # Massage argv[0]: + # - Convert portable slashes ('/') to the flavor preferred by the + # OS we're currently running on. + # - Run python script thru the current python interpreter (important + # on systems that doesn't sport native hash-bang script execution). + # + asArgs[0] = asArgs[0].replace('/', os.path.sep); + if not os.path.isabs(asArgs[0]): + asArgs[0] = os.path.join(self._oTestBoxScript.getPathScripts(), asArgs[0]); + + if asArgs[0].endswith('.py') and fWithInterpreter: + if sys.executable: + asArgs.insert(0, sys.executable); + else: + asArgs.insert(0, 'python'); + + return asArgs; + + def _outputThreadProc(self, oChild, oStdOut, sAction): + """ + Thread procedure for the thread that reads the output of the child + process. We use a dedicated thread for this purpose since non-blocking + I/O may be hard to keep portable according to hints around the web... + """ + oThread = oChild.oOutputThread; + while not oThread.fPleaseQuit: + # Get a line. + try: + sLine = oStdOut.readline(); + except Exception as oXcpt: + self._log('child (%s) pipe I/O error: %s' % (sAction, oXcpt,)); + break; + + # EOF? + if not sLine: + break; + + # Strip trailing new line (DOS and UNIX). + if sLine.endswith("\r\n"): + sLine = sLine[0:-2]; + elif sLine.endswith("\n"): + sLine = sLine[0:-1]; + + # Log it. + if self._logInternal(sLine, fPrefix = False, fFlushCheck = True): + self._logFlush(); + + # Close the stdout pipe in case we were told to get lost. + try: + oStdOut.close(); + except Exception as oXcpt: + self._log('warning: Exception closing stdout pipe of "%s" child: %s' % (sAction, oXcpt,)); + + # This is a bit hacky, but try reap the child so it won't hang as + # defunkt during abort/timeout. + if oChild.poll() is None: + for _ in range(15): + time.sleep(0.2); + if oChild.poll() is not None: + break; + + oChild = None; + return None; + + def _spawnChild(self, sAction): + """ + Spawns the child process, returning success indicator + child object. + """ + + # Argument list. + asArgs = self._assembleArguments(sAction) + if asArgs is None: + self._log('Malformed command line: "%s"' % (self._sScriptCmdLine,)); + return (False, None); + + # Spawn child. + try: + oChild = utils.processPopenSafe(asArgs, + shell = False, + bufsize = -1, + stdout = subprocess.PIPE, + stderr = subprocess.STDOUT, + cwd = self._oTestBoxScript.getPathSpill(), + universal_newlines = True, + close_fds = utils.getHostOs() != 'win', + preexec_fn = (None if utils.getHostOs() in ['win', 'os2'] + else os.setsid)); # pylint: disable=no-member + except Exception as oXcpt: + self._log('Error creating child process %s: %s' % (asArgs, oXcpt)); + return (False, None); + + oChild.sTestBoxScriptAction = sAction; + + # Start output thread, extending the child object to keep track of it. + oChild.oOutputThread = threading.Thread(target=self._outputThreadProc, args=(oChild, oChild.stdout, sAction)) + oChild.oOutputThread.daemon = True; + oChild.oOutputThread.fPleaseQuit = False; # Our extension. + oChild.oOutputThread.start(); + + return (True, oChild); + + def _monitorChild(self, cSecTimeout, fTryKillCommand = True, oChild = None): + """ + Monitors the child process. If the child executes longer that + cSecTimeout allows, we'll terminate it. + Returns Success indicator and constants.result value. + """ + + if oChild is None: + oChild = self._oChild; + + iProcGroup = oChild.pid; + if utils.getHostOs() in ['win', 'os2'] or iProcGroup <= 0: + iProcGroup = -2; + + # + # Do timeout processing and check the health of the child. + # + sResult = constants.result.PASSED; + seStarted = utils.timestampSecond(); + while True: + # Check status. + iRc = oChild.poll(); + if iRc is not None: + self._log('Child doing "%s" completed with exit code %d' % (oChild.sTestBoxScriptAction, iRc)); + oChild.oOutputThread.join(self.kcSecFinalOutputTimeout); + + if oChild is self._oChild: + self._oChild = None; + + if iRc == constants.rtexitcode.SUCCESS: + return (True, constants.result.PASSED); + if iRc == constants.rtexitcode.SKIPPED: + return (True, constants.result.SKIPPED); + if iRc == constants.rtexitcode.BAD_TESTBOX: + return (False, constants.result.BAD_TESTBOX); + return (False, constants.result.FAILED); + + # Check for abort first, since that has less of a stigma. + if self._shouldTerminate() is True: + sResult = constants.result.ABORTED; + break; + + # Check timeout. + cSecElapsed = utils.timestampSecond() - seStarted; + if cSecElapsed > cSecTimeout: + self._log('Timeout: %u secs (limit %u secs)' % (cSecElapsed, cSecTimeout)); + sResult = constants.result.TIMED_OUT; + break; + + # Wait. + cSecLeft = cSecTimeout - cSecElapsed; + oChild.oOutputThread.join(15 if cSecLeft > 15 else (cSecLeft + 1)); + + # + # If the child is still alive, try use the abort command to stop it + # very gently. This let's the testdriver clean up daemon processes + # and such that our code below won't catch. + # + if fTryKillCommand and oChild.poll() is None: + self._log('Attempting to abort child...'); + (fRc2, oAbortChild) = self._spawnChild('abort'); + if oAbortChild is not None and fRc2 is True: + self._monitorChild(self.kcSecAbortTimeout, False, oAbortChild); + oAbortChild = None; + + # + # If the child is still alive, try the polite way. + # + if oChild.poll() is None: + self._log('Attempting to terminate child doing "%s"...' % (oChild.sTestBoxScriptAction,)); + + if iProcGroup > 0: + try: + os.killpg(iProcGroup, signal.SIGTERM); # pylint: disable=no-member + except Exception as oXcpt: + self._log('killpg() failed: %s' % (oXcpt,)); + + try: + self._oChild.terminate(); + oChild.oOutputThread.join(self.kcSecTerminateOutputTimeout); + except Exception as oXcpt: + self._log('terminate() failed: %s' % (oXcpt,)); + + # + # If the child doesn't respond to polite, kill it. Always do a killpg + # should there be any processes left in the group. + # + if iProcGroup > 0: + try: + os.killpg(iProcGroup, signal.SIGKILL); # pylint: disable=no-member + except Exception as oXcpt: + self._log('killpg() failed: %s' % (oXcpt,)); + + if oChild.poll() is None: + self._log('Attemting to kill child doing "%s"...' % (oChild.sTestBoxScriptAction,)); + try: + self._oChild.kill(); + oChild.oOutputThread.join(self.kcSecKillOutputTimeout); + except Exception as oXcpt: + self._log('kill() failed: %s' % (oXcpt,)); + + # + # Give the whole mess a couple of more seconds to respond in case the + # output thread exitted prematurely for some weird reason. + # + if oChild.poll() is None: + time.sleep(2); + time.sleep(2); + time.sleep(2); + + iRc = oChild.poll(); + if iRc is not None: + self._log('Child doing "%s" aborted with exit code %d' % (oChild.sTestBoxScriptAction, iRc)); + else: + self._log('Child doing "%s" is still running, giving up...' % (oChild.sTestBoxScriptAction,)); + ## @todo in this case we should probably try reboot the testbox... + oChild.oOutputThread.fPleaseQuit = True; + + if oChild is self._oChild: + self._oChild = None; + return (False, sResult); + + def _terminateChild(self): + """ + Terminates the child forcefully. + """ + if self._oChild is not None: + pass; + + def _cleanupAfter(self): + """ + Cleans up after a test failure. (On success, cleanup is implicit.) + """ + assert self._oChild is None; + + # + # Tell the script to clean up. + # + if self._sScriptCmdLine: # can be empty if cleanup crashed. + (fRc, self._oChild) = self._spawnChild('cleanup-after'); + if fRc is True: + (fRc, _) = self._monitorChild(self.kcSecCleanupTimeout, False); + self._terminateChild(); + else: + fRc = False; + + # + # Wipe the stuff clean. + # + fRc2 = self._oTestBoxScript.reinitScratch(fnLog = self._log, cRetries = 6); + + return fRc and fRc2; + + + +class TestBoxCleanupTask(TestBoxTestDriverTask): + """ + Special asynchronous task for cleaning up a stale test when starting the + testbox script. It's assumed that the reason for the stale test lies in + it causing a panic, reboot, or similar, so we'll also try collect some + info about recent system crashes and reboots. + """ + + def __init__(self, oTestBoxScript): + # Read the old state, throwing a fit if it's invalid. + sScriptState = oTestBoxScript.getPathState(); + sScriptCmdLine = self._readStateFile(os.path.join(sScriptState, 'script-cmdline.txt')); + sResultId = self._readStateFile(os.path.join(sScriptState, 'result-id.txt')); + try: + idResult = int(sResultId); + if idResult <= 0 or idResult >= 0x7fffffff: + raise Exception(''); + except: + raise Exception('Invalid id value "%s" found in %s' % (sResultId, os.path.join(sScriptState, 'result-id.txt'))); + + sTestBoxId = self._readStateFile(os.path.join(sScriptState, 'testbox-id.txt')); + try: + self.idTestBox = int(sTestBoxId); + if self.idTestBox <= 0 or self.idTestBox >= 0x7fffffff: + raise Exception(''); + except: + raise Exception('Invalid id value "%s" found in %s' % (sTestBoxId, os.path.join(sScriptState, 'testbox-id.txt'))); + self.sTestBoxName = self._readStateFile(os.path.join(sScriptState, 'testbox-name.txt')); + + # Init super. + TestBoxTestDriverTask.__init__(self, oTestBoxScript, self._threadProc, self.kcSecCleanupTimeout, + idResult, sScriptCmdLine); + + @staticmethod + def _readStateFile(sPath): + """ + Reads a state file, returning a string on success and otherwise raising + an exception. + """ + try: + with open(sPath, "rb") as oFile: + sStr = oFile.read(); + sStr = sStr.decode('utf-8'); + return sStr.strip(); + except Exception as oXcpt: + raise Exception('Failed to read "%s": %s' % (sPath, oXcpt)); + + def _threadProc(self): + """ + Perform the actual clean up on script startup. + """ + + # + # First make sure we won't repeat this exercise should it turn out to + # trigger another reboot/panic/whatever. + # + sScriptCmdLine = os.path.join(self._oTestBoxScript.getPathState(), 'script-cmdline.txt'); + try: + os.remove(sScriptCmdLine); + open(sScriptCmdLine, 'wb').close(); # pylint: disable=consider-using-with + except Exception as oXcpt: + self._log('Error truncating "%s": %s' % (sScriptCmdLine, oXcpt)); + + # + # Report the incident. + # + self._log('Seems we rebooted!'); + self._log('script-cmdline="%s"' % (self._sScriptCmdLine)); + self._log('result-id=%d' % (self._idResult)); + self._log('testbox-id=%d' % (self.idTestBox)); + self._log('testbox-name=%s' % (self.sTestBoxName)); + self._logFlush(); + + # System specific info. + sOs = utils.getHostOs(); + if sOs == 'darwin': + self._log('NVRAM Panic Info:\n%s\n' % (self.darwinGetPanicInfo(),)); + + self._logFlush(); + ## @todo Add some special command for reporting this situation so we get something + # useful in the event log. + + # + # Do the cleaning up. + # + self._cleanupAfter(); + + self._reportDone(constants.result.REBOOTED); + return False; + + def darwinGetPanicInfo(self): + """ + Returns a string with the aapl,panic-info content. + """ + # Retriev the info. + try: + sRawInfo = utils.processOutputChecked(['nvram', 'aapl,panic-info']); + except Exception as oXcpt: + return 'exception running nvram: %s' % (oXcpt,); + + # Decode (%xx) and decompact it (7-bit -> 8-bit). + ahDigits = \ + { + '0': 0, '1': 1, '2': 2, '3': 3, '4': 4, '5': 5, '6': 6, '7': 7, + '8': 8, '9': 9, 'a': 10, 'b': 11, 'c': 12, 'd': 13, 'e': 14, 'f': 15, + }; + sInfo = ''; + off = len('aapl,panic-info') + 1; + iBit = 0; + bLow = 0; + + while off < len(sRawInfo): + # isprint is used to determine whether to %xx or %c it, so we have to + # be a little careful before assuming % sequences are hex bytes. + if sRawInfo[off] == '%' \ + and off + 3 <= len(sRawInfo) \ + and sRawInfo[off + 1] in ahDigits \ + and sRawInfo[off + 2] in ahDigits: + bCur = ahDigits[sRawInfo[off + 1]] * 0x10 + ahDigits[sRawInfo[off + 2]]; + off += 3; + else: + bCur = ord(sRawInfo[off]); + off += 1; + + sInfo += chr(((bCur & (0x7f >> iBit)) << iBit) | bLow); + bLow = bCur >> (7 - iBit); + + if iBit < 6: + iBit += 1; + else: + # Final bit in sequence. + sInfo += chr(bLow); + bLow = 0; + iBit = 0; + + # Expand shorthand. + sInfo = sInfo.replace('@', 'com.apple.'); + sInfo = sInfo.replace('>', 'com.apple.driver.'); + sInfo = sInfo.replace('|', 'com.apple.iokit.'); + sInfo = sInfo.replace('$', 'com.apple.security.'); + sInfo = sInfo.replace('!A', 'Apple'); + sInfo = sInfo.replace('!a', 'Action'); + sInfo = sInfo.replace('!B', 'Bluetooth'); + sInfo = sInfo.replace('!C', 'Controller'); + sInfo = sInfo.replace('!F', 'Family'); + sInfo = sInfo.replace('!I', 'Intel'); + sInfo = sInfo.replace('!U', 'AppleUSB'); + sInfo = sInfo.replace('!P', 'Profile'); + + # Done. + return sInfo + + +class TestBoxExecTask(TestBoxTestDriverTask): + """ + Implementation of a asynchronous EXEC task. + + This uses a thread for doing the actual work, i.e. starting and monitoring + the child process, processing its output, and more. + """ + + def __init__(self, oTestBoxScript, idResult, sScriptZips, sScriptCmdLine, cSecTimeout): + """ + Class instance init + """ + # Init our instance data. + self._sScriptZips = sScriptZips; + + # Init super. + TestBoxTestDriverTask.__init__(self, oTestBoxScript, self._threadProc, cSecTimeout, idResult, sScriptCmdLine); + + @staticmethod + def _writeStateFile(sPath, sContent): + """ + Writes a state file, raising an exception on failure. + """ + try: + with open(sPath, "wb") as oFile: + oFile.write(sContent.encode('utf-8')); + oFile.flush(); + try: os.fsync(oFile.fileno()); + except: pass; + except Exception as oXcpt: + raise Exception('Failed to write "%s": %s' % (sPath, oXcpt)); + return True; + + @staticmethod + def _environTxtContent(): + """ + Collects environment variables and values for the environ.txt stat file + (for external monitoring tool). + """ + sText = ''; + for sVar in [ 'TESTBOX_PATH_BUILDS', 'TESTBOX_PATH_RESOURCES', 'TESTBOX_PATH_SCRATCH', 'TESTBOX_PATH_SCRIPTS', + 'TESTBOX_PATH_UPLOAD', 'TESTBOX_HAS_HW_VIRT', 'TESTBOX_HAS_NESTED_PAGING', 'TESTBOX_HAS_IOMMU', + 'TESTBOX_SCRIPT_REV', 'TESTBOX_CPU_COUNT', 'TESTBOX_MEM_SIZE', 'TESTBOX_SCRATCH_SIZE', + 'TESTBOX_WITH_RAW_MODE', 'TESTBOX_WITH_RAW_MODE', 'TESTBOX_MANAGER_URL', 'TESTBOX_UUID', + 'TESTBOX_REPORTER', 'TESTBOX_NAME', 'TESTBOX_ID', 'TESTBOX_TEST_SET_ID', + 'TESTBOX_TIMEOUT', 'TESTBOX_TIMEOUT_ABS', ]: + sValue = os.environ.get(sVar); + if sValue: + sText += sVar + '=' + sValue + '\n'; + return sText; + + def _saveState(self): + """ + Saves the task state on disk so we can launch a TestBoxCleanupTask job + if the test should cause system panic or similar. + + Note! May later be extended to support tests that reboots the host. + """ + sScriptState = self._oTestBoxScript.getPathState(); + try: + self._writeStateFile(os.path.join(sScriptState, 'script-cmdline.txt'), self._sScriptCmdLine); + self._writeStateFile(os.path.join(sScriptState, 'result-id.txt'), str(self._idResult)); + self._writeStateFile(os.path.join(sScriptState, 'testbox-id.txt'), str(self._oTestBoxScript.getTestBoxId())); + self._writeStateFile(os.path.join(sScriptState, 'testbox-name.txt'), self._oTestBoxScript.getTestBoxName()); + self._writeStateFile(os.path.join(sScriptState, 'environ.txt'), self._environTxtContent()); + except Exception as oXcpt: + self._log('Failed to write state: %s' % (oXcpt,)); + return False; + return True; + + def _downloadAndUnpackScriptZips(self): + """ + Downloads/copies the script ZIPs into TESTBOX_SCRIPT and unzips them to + the same directory. + + Raises no exceptions, returns log + success indicator instead. + """ + sPathScript = self._oTestBoxScript.getPathScripts(); + asArchives = self._sScriptZips.split(','); + for sArchive in asArchives: + sArchive = sArchive.strip(); + if not sArchive: + continue; + + # Figure the destination name (in scripts). + sDstFile = webutils.getFilename(sArchive); + if not sDstFile \ + or re.search('[^a-zA-Z0-9 !#$%&\'()@^_`{}~.-]', sDstFile) is not None: # FAT charset sans 128-255 + '.'. + self._log('Malformed script zip filename: %s' % (sArchive,)); + return False; + sDstFile = os.path.join(sPathScript, sDstFile); + + # Do the work. + if webutils.downloadFile(sArchive, sDstFile, self._oTestBoxScript.getPathBuilds(), self._log, self._log) is not True: + return False; + asFiles = utils.unpackFile(sDstFile, sPathScript, self._log, self._log); + if asFiles is None: + return False; + + # Since zip files doesn't always include mode masks, set the X bit + # of all of them so we can execute binaries and hash-bang scripts. + for sFile in asFiles: + utils.chmodPlusX(sFile); + + return True; + + def _threadProc(self): + """ + Do the work of an EXEC command. + """ + + sResult = constants.result.PASSED; + + # + # Start by preparing the scratch directories. + # + # Note! Failures at this stage are not treated as real errors since + # they may be caused by the previous test and other circumstances + # so we don't want to go fail a build because of this. + # + fRc = self._oTestBoxScript.reinitScratch(self._logInternal); + fNeedCleanUp = fRc; + if fRc is True: + fRc = self._downloadAndUnpackScriptZips(); + testboxcommons.log2('_threadProc: _downloadAndUnpackScriptZips -> %s' % (fRc,)); + if fRc is not True: + sResult = constants.result.BAD_TESTBOX; + + # + # Make sure the script exists. + # + if fRc is True: + sScript = self._assembleArguments('none', fWithInterpreter = False)[0]; + if not os.path.exists(sScript): + self._log('The test driver script "%s" cannot be found.' % (sScript,)); + sDir = sScript; + while len(sDir) > 3: + sDir = os.path.dirname(sDir); + if os.path.exists(sDir): + self._log('First existing parent directory is "%s".' % (sDir,)); + break; + fRc = False; + + if fRc is True: + # + # Start testdriver script. + # + fRc = self._saveState(); + if fRc: + (fRc, self._oChild) = self._spawnChild('all'); + testboxcommons.log2('_threadProc: _spawnChild -> %s, %s' % (fRc, self._oChild)); + if fRc: + (fRc, sResult) = self._monitorChild(self._cSecTimeout); + testboxcommons.log2('_threadProc: _monitorChild -> %s' % (fRc,)); + + # If the run failed, do explicit cleanup unless its a BAD_TESTBOX, since BAD_TESTBOX is + # intended for pre-cleanup problems caused by previous test failures. Do a cleanup on + # a BAD_TESTBOX could easily trigger an uninstallation error and change status to FAILED. + if fRc is not True: + if sResult != constants.result.BAD_TESTBOX: + testboxcommons.log2('_threadProc: explicit cleanups...'); + self._terminateChild(); + self._cleanupAfter(); + fNeedCleanUp = False; + assert self._oChild is None; + + # + # Clean up scratch. + # + if fNeedCleanUp: + if self._oTestBoxScript.reinitScratch(self._logInternal, cRetries = 6) is not True: + self._log('post run reinitScratch failed.'); + fRc = False; + + # + # Report status and everything back to the test manager. + # + if fRc is False and sResult == constants.result.PASSED: + sResult = constants.result.FAILED; + self._reportDone(sResult); + return fRc; + |