diff options
Diffstat (limited to 'src/VBox/ValidationKit/testmanager/batch')
9 files changed, 2473 insertions, 0 deletions
diff --git a/src/VBox/ValidationKit/testmanager/batch/Makefile.kmk b/src/VBox/ValidationKit/testmanager/batch/Makefile.kmk new file mode 100644 index 00000000..6f03da64 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/batch/Makefile.kmk @@ -0,0 +1,36 @@ +# $Id: Makefile.kmk $ +## @file +# VirtualBox Validation Kit. +# + +# +# Copyright (C) 2006-2019 Oracle Corporation +# +# This file is part of VirtualBox Open Source Edition (OSE), as +# available from http://www.virtualbox.org. This file is free software; +# you can redistribute it and/or modify it under the terms of the GNU +# General Public License (GPL) as published by the Free Software +# Foundation, in version 2 as it comes in the "COPYING" file of the +# VirtualBox OSE distribution. VirtualBox OSE is distributed in the +# hope that it will be useful, but WITHOUT ANY WARRANTY of any kind. +# +# The contents of this file may alternatively be used under the terms +# of the Common Development and Distribution License Version 1.0 +# (CDDL) only, as it comes in the "COPYING.CDDL" file of the +# VirtualBox OSE distribution, in which case the provisions of the +# CDDL are applicable instead of those of the GPL. +# +# You may elect to license modified versions of this file under the +# terms and conditions of either the GPL or the CDDL or both. +# + +SUB_DEPTH = ../../../../.. +include $(KBUILD_PATH)/subheader.kmk + + +VBOX_VALIDATIONKIT_PYTHON_SOURCES += $(wildcard $(PATH_SUB_CURRENT)/*.py) + +$(evalcall def_vbox_validationkit_process_python_sources) +$(evalcall def_vbox_validationkit_process_js_sources) +include $(FILE_KBUILD_SUB_FOOTER) + diff --git a/src/VBox/ValidationKit/testmanager/batch/add_build.py b/src/VBox/ValidationKit/testmanager/batch/add_build.py new file mode 100755 index 00000000..0cdb8b45 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/batch/add_build.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# $Id: add_build.py $ +# pylint: disable=C0301 + +""" +Interface used by the tinderbox server side software to add a fresh build. +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2019 Oracle Corporation + +This file is part of VirtualBox Open Source Edition (OSE), as +available from http://www.virtualbox.org. This file is free software; +you can redistribute it and/or modify it under the terms of the GNU +General Public License (GPL) as published by the Free Software +Foundation, in version 2 as it comes in the "COPYING" file of the +VirtualBox OSE distribution. VirtualBox OSE is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY of any kind. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL) only, as it comes in the "COPYING.CDDL" file of the +VirtualBox OSE distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. +""" +__version__ = "$Revision: 127855 $" + +# Standard python imports +import sys; +import os; +from optparse import OptionParser; # pylint: disable=deprecated-module + +# Add Test Manager's modules path +g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))); +sys.path.append(g_ksTestManagerDir); + +# Test Manager imports +from testmanager.core.db import TMDatabaseConnection; +from testmanager.core.build import BuildDataEx, BuildLogic, BuildCategoryData; + +class Build(object): # pylint: disable=R0903 + """ + Add build info into Test Manager database. + """ + + def __init__(self): + """ + Parse command line. + """ + + oParser = OptionParser(); + oParser.add_option('-q', '--quiet', dest = 'fQuiet', action = 'store_true', + help = 'Quiet execution'); + oParser.add_option('-b', '--branch', dest = 'sBranch', metavar = '<branch>', + help = 'branch name (default: trunk)', default = 'trunk'); + oParser.add_option('-p', '--product', dest = 'sProductName', metavar = '<name>', + help = 'The product name.'); + oParser.add_option('-r', '--revision', dest = 'iRevision', metavar = '<rev>', + help = 'revision number'); + oParser.add_option('-R', '--repository', dest = 'sRepository', metavar = '<repository>', + help = 'Version control repository name.'); + oParser.add_option('-t', '--type', dest = 'sBuildType', metavar = '<type>', + help = 'build type (debug, release etc.)'); + oParser.add_option('-v', '--version', dest = 'sProductVersion', metavar = '<ver>', + help = 'The product version number (suitable for RTStrVersionCompare)'); + oParser.add_option('-o', '--os-arch', dest = 'asTargetOsArches', metavar = '<os.arch>', action = 'append', + help = 'Target OS and architecture. This option can be repeated.'); + oParser.add_option('-l', '--log', dest = 'sBuildLogPath', metavar = '<url>', + help = 'URL to the build logs (optional).'); + oParser.add_option('-f', '--file', dest = 'asFiles', metavar = '<file|url>', action = 'append', + help = 'URLs or build share relative path to a build output file. This option can be repeated.'); + + (self.oConfig, _) = oParser.parse_args(); + + # Check command line + asMissing = []; + if self.oConfig.sBranch is None: asMissing.append('--branch'); + if self.oConfig.iRevision is None: asMissing.append('--revision'); + if self.oConfig.sProductVersion is None: asMissing.append('--version'); + if self.oConfig.sProductName is None: asMissing.append('--product'); + if self.oConfig.sBuildType is None: asMissing.append('--type'); + if self.oConfig.asTargetOsArches is None: asMissing.append('--os-arch'); + if self.oConfig.asFiles is None: asMissing.append('--file'); + if asMissing: + sys.stderr.write('syntax error: Missing: %s\n' % (asMissing,)); + sys.exit(1); + # Temporary default. + if self.oConfig.sRepository is None: + self.oConfig.sRepository = 'vbox'; + + def add(self): + """ + Add build data record into database. + """ + oDb = TMDatabaseConnection() + + # Assemble the build data. + oBuildData = BuildDataEx() + oBuildData.idBuildCategory = None; + oBuildData.iRevision = self.oConfig.iRevision + oBuildData.sVersion = self.oConfig.sProductVersion + oBuildData.sLogUrl = self.oConfig.sBuildLogPath + oBuildData.sBinaries = ','.join(self.oConfig.asFiles); + oBuildData.oCat = BuildCategoryData().initFromValues(sProduct = self.oConfig.sProductName, + sRepository = self.oConfig.sRepository, + sBranch = self.oConfig.sBranch, + sType = self.oConfig.sBuildType, + asOsArches = self.oConfig.asTargetOsArches); + + # Add record to database + try: + BuildLogic(oDb).addEntry(oBuildData, fCommit = True); + except: + if self.oConfig.fQuiet: + sys.exit(1); + raise; + oDb.close(); + return 0; + +if __name__ == '__main__': + sys.exit(Build().add()); + diff --git a/src/VBox/ValidationKit/testmanager/batch/check_for_deleted_builds.py b/src/VBox/ValidationKit/testmanager/batch/check_for_deleted_builds.py new file mode 100755 index 00000000..8d12ff20 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/batch/check_for_deleted_builds.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# $Id: check_for_deleted_builds.py $ +# pylint: disable=C0301 + +""" +Admin job for checking detecting deleted builds. + +This is necessary when the tinderbox <-> test manager interface was +busted and the build info in is out of sync. The result is generally +a lot of skipped tests because of missing builds, typically during +bisecting problems. +""" + +from __future__ import print_function; + +__copyright__ = \ +""" +Copyright (C) 2012-2019 Oracle Corporation + +This file is part of VirtualBox Open Source Edition (OSE), as +available from http://www.virtualbox.org. This file is free software; +you can redistribute it and/or modify it under the terms of the GNU +General Public License (GPL) as published by the Free Software +Foundation, in version 2 as it comes in the "COPYING" file of the +VirtualBox OSE distribution. VirtualBox OSE is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY of any kind. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL) only, as it comes in the "COPYING.CDDL" file of the +VirtualBox OSE distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. +""" +__version__ = "$Revision: 127855 $" + +# Standard python imports +import sys; +import os; +from optparse import OptionParser; # pylint: disable=deprecated-module + +# Add Test Manager's modules path +g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))); +sys.path.append(g_ksTestManagerDir); + +# Test Manager imports +from testmanager.core.db import TMDatabaseConnection; +from testmanager.core.build import BuildLogic; + + + +class BuildChecker(object): # pylint: disable=R0903 + """ + Add build info into Test Manager database. + """ + + def __init__(self): + """ + Parse command line. + """ + + oParser = OptionParser(); + oParser.add_option('-q', '--quiet', dest = 'fQuiet', action = 'store_true', default = False, + help = 'Quiet execution'); + oParser.add_option('--dry-run', dest = 'fRealRun', action = 'store_false', default = False, + help = 'Dry run'); + oParser.add_option('--real-run', dest = 'fRealRun', action = 'store_true', default = False, + help = 'Real run'); + + (self.oConfig, _) = oParser.parse_args(); + if not self.oConfig.fQuiet: + if not self.oConfig.fRealRun: + print('Dry run.'); + else: + print('Real run! Will commit findings!'); + + + def checkBuilds(self): + """ + Add build data record into database. + """ + oDb = TMDatabaseConnection(); + oBuildLogic = BuildLogic(oDb); + + tsNow = oDb.getCurrentTimestamp(); + cMaxRows = 1024; + iStart = 0; + while True: + aoBuilds = oBuildLogic.fetchForListing(iStart, cMaxRows, tsNow); + if not self.oConfig.fQuiet and aoBuilds: + print('Processing builds #%s thru #%s' % (aoBuilds[0].idBuild, aoBuilds[-1].idBuild)); + + for oBuild in aoBuilds: + if oBuild.fBinariesDeleted is False: + rc = oBuild.areFilesStillThere(); + if rc is False: + if not self.oConfig.fQuiet: + print('missing files for build #%s / r%s / %s / %s / %s / %s / %s' + % (oBuild.idBuild, oBuild.iRevision, oBuild.sVersion, oBuild.oCat.sType, + oBuild.oCat.sBranch, oBuild.oCat.sProduct, oBuild.oCat.asOsArches,)); + print(' %s' % (oBuild.sBinaries,)); + if self.oConfig.fRealRun is True: + oBuild.fBinariesDeleted = True; + oBuildLogic.editEntry(oBuild, fCommit = True); + elif rc is True and not self.oConfig.fQuiet: + print('build #%s still have its files' % (oBuild.idBuild,)); + elif rc is None and not self.oConfig.fQuiet: + print('Unable to determine state of build #%s' % (oBuild.idBuild,)); + + # advance + if len(aoBuilds) < cMaxRows: + break; + iStart += len(aoBuilds); + + oDb.close(); + return 0; + +if __name__ == '__main__': + sys.exit(BuildChecker().checkBuilds()); + diff --git a/src/VBox/ValidationKit/testmanager/batch/close_orphaned_testsets.py b/src/VBox/ValidationKit/testmanager/batch/close_orphaned_testsets.py new file mode 100755 index 00000000..331e3125 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/batch/close_orphaned_testsets.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# $Id: close_orphaned_testsets.py $ +# pylint: disable=C0301 + +""" +Maintenance tool for closing orphaned testsets. +""" + +from __future__ import print_function; + +__copyright__ = \ +""" +Copyright (C) 2012-2019 Oracle Corporation + +This file is part of VirtualBox Open Source Edition (OSE), as +available from http://www.virtualbox.org. This file is free software; +you can redistribute it and/or modify it under the terms of the GNU +General Public License (GPL) as published by the Free Software +Foundation, in version 2 as it comes in the "COPYING" file of the +VirtualBox OSE distribution. VirtualBox OSE is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY of any kind. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL) only, as it comes in the "COPYING.CDDL" file of the +VirtualBox OSE distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. +""" +__version__ = "$Revision: 127855 $" + +# Standard python imports +import sys +import os +from optparse import OptionParser; # pylint: disable=deprecated-module + +# Add Test Manager's modules path +g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.append(g_ksTestManagerDir) + +# Test Manager imports +from testmanager.core.db import TMDatabaseConnection +from testmanager.core.testset import TestSetLogic; + + +class CloseOrphanedTestSets(object): + """ + Finds and closes orphaned testsets. + """ + + def __init__(self): + """ + Parse command line + """ + oParser = OptionParser(); + oParser.add_option('-d', '--just-do-it', dest='fJustDoIt', action='store_true', + help='Do the database changes.'); + + + (self.oConfig, _) = oParser.parse_args(); + + + def main(self): + """ Main method. """ + oDb = TMDatabaseConnection(); + + # Get a list of orphans. + oLogic = TestSetLogic(oDb); + aoOrphans = oLogic.fetchOrphaned(); + if aoOrphans: + # Complete them. + if self.oConfig.fJustDoIt: + print('Completing %u test sets as abandoned:' % (len(aoOrphans),)); + for oTestSet in aoOrphans: + print('#%-7u: idTestBox=%-3u tsCreated=%s tsDone=%s' + % (oTestSet.idTestSet, oTestSet.idTestBox, oTestSet.tsCreated, oTestSet.tsDone)); + oLogic.completeAsAbandoned(oTestSet.idTestSet); + print('Committing...'); + oDb.commit(); + else: + for oTestSet in aoOrphans: + print('#%-7u: idTestBox=%-3u tsCreated=%s tsDone=%s' + % (oTestSet.idTestSet, oTestSet.idTestBox, oTestSet.tsCreated, oTestSet.tsDone)); + print('Not completing any testsets without seeing the --just-do-it option.'); + else: + print('No orphaned test sets.\n'); + return 0; + + +if __name__ == '__main__': + sys.exit(CloseOrphanedTestSets().main()) + diff --git a/src/VBox/ValidationKit/testmanager/batch/del_build.py b/src/VBox/ValidationKit/testmanager/batch/del_build.py new file mode 100755 index 00000000..252eb59d --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/batch/del_build.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# $Id: del_build.py $ +# pylint: disable=C0301 + +""" +Interface used by the tinderbox server side software to mark build binaries +deleted. +""" + +from __future__ import print_function; + +__copyright__ = \ +""" +Copyright (C) 2012-2019 Oracle Corporation + +This file is part of VirtualBox Open Source Edition (OSE), as +available from http://www.virtualbox.org. This file is free software; +you can redistribute it and/or modify it under the terms of the GNU +General Public License (GPL) as published by the Free Software +Foundation, in version 2 as it comes in the "COPYING" file of the +VirtualBox OSE distribution. VirtualBox OSE is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY of any kind. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL) only, as it comes in the "COPYING.CDDL" file of the +VirtualBox OSE distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. +""" +__version__ = "$Revision: 127855 $" + +# Standard python imports +import sys +import os +from optparse import OptionParser; # pylint: disable=deprecated-module + +# Add Test Manager's modules path +g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.append(g_ksTestManagerDir) + +# Test Manager imports +from testmanager.core.db import TMDatabaseConnection +from testmanager.core.build import BuildLogic + + +def markBuildsDeleted(): + """ + Marks the builds using the specified binaries as deleted. + """ + + oParser = OptionParser() + oParser.add_option('-q', '--quiet', dest='fQuiet', action='store_true', + help='Quiet execution'); + + (oConfig, asArgs) = oParser.parse_args() + if not asArgs: + if not oConfig.fQuiet: + sys.stderr.write('syntax error: No builds binaries specified\n'); + return 1; + + + oDb = TMDatabaseConnection() + oLogic = BuildLogic(oDb) + + for sBuildBin in asArgs: + try: + cBuilds = oLogic.markDeletedByBinaries(sBuildBin, fCommit = True) + except: + if oConfig.fQuiet: + sys.exit(1); + raise; + else: + if not oConfig.fQuiet: + print("del_build.py: Marked %u builds associated with '%s' as deleted." % (cBuilds, sBuildBin,)); + + oDb.close() + return 0; + +if __name__ == '__main__': + sys.exit(markBuildsDeleted()) + diff --git a/src/VBox/ValidationKit/testmanager/batch/filearchiver.py b/src/VBox/ValidationKit/testmanager/batch/filearchiver.py new file mode 100755 index 00000000..9c6bb8e8 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/batch/filearchiver.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# $Id: filearchiver.py $ +# pylint: disable=C0301 + +""" +A cronjob that compresses logs and other files, moving them to the +g_ksZipFileAreaRootDir storage area. +""" + +from __future__ import print_function; + +__copyright__ = \ +""" +Copyright (C) 2012-2019 Oracle Corporation + +This file is part of VirtualBox Open Source Edition (OSE), as +available from http://www.virtualbox.org. This file is free software; +you can redistribute it and/or modify it under the terms of the GNU +General Public License (GPL) as published by the Free Software +Foundation, in version 2 as it comes in the "COPYING" file of the +VirtualBox OSE distribution. VirtualBox OSE is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY of any kind. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL) only, as it comes in the "COPYING.CDDL" file of the +VirtualBox OSE distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. +""" +__version__ = "$Revision: 127855 $" + +# Standard python imports +import sys +import os +from optparse import OptionParser; # pylint: disable=deprecated-module +import time; +import zipfile; + +# Add Test Manager's modules path +g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.append(g_ksTestManagerDir) + +# Test Manager imports +from common import utils; +from testmanager import config; +from testmanager.core.db import TMDatabaseConnection; +from testmanager.core.testset import TestSetData, TestSetLogic; + + + +class FileArchiverBatchJob(object): # pylint: disable=R0903 + """ + Log+files comp + """ + + def __init__(self, oOptions): + """ + Parse command line + """ + self.fVerbose = oOptions.fVerbose; + self.sSrcDir = config.g_ksFileAreaRootDir; + self.sDstDir = config.g_ksZipFileAreaRootDir; + #self.oTestSetLogic = TestSetLogic(TMDatabaseConnection(self.dprint if self.fVerbose else None)); + self.oTestSetLogic = TestSetLogic(TMDatabaseConnection(None)); + self.fDryRun = oOptions.fDryRun; + + def dprint(self, sText): + """ Verbose output. """ + if self.fVerbose: + print(sText); + return True; + + def warning(self, sText): + """Prints a warning.""" + print(sText); + return True; + + def _processTestSet(self, idTestSet, asFiles, sCurDir): + """ + Worker for processDir. + Same return codes as processDir. + """ + + sBaseFilename = os.path.join(sCurDir, 'TestSet-%d' % (idTestSet,)); + if sBaseFilename[0:2] == ('.' + os.path.sep): + sBaseFilename = sBaseFilename[2:]; + sSrcFileBase = os.path.join(self.sSrcDir, sBaseFilename + '-'); + + # + # Skip the file if the test set is still running. + # But delete them if the testset is not found. + # + oTestSet = self.oTestSetLogic.tryFetch(idTestSet); + if oTestSet is not None and sBaseFilename != oTestSet.sBaseFilename: + self.warning('TestSet %d: Deleting because sBaseFilename differs: "%s" (disk) vs "%s" (db)' \ + % (idTestSet, sBaseFilename, oTestSet.sBaseFilename,)); + oTestSet = None; + + if oTestSet is not None: + if oTestSet.enmStatus == TestSetData.ksTestStatus_Running: + self.dprint('Skipping test set #%d, still running' % (idTestSet,)); + return True; + + # + # If we have a zip file already, don't try recreate it as we might + # have had trouble removing the source files. + # + sDstDirPath = os.path.join(self.sDstDir, sCurDir); + sZipFileNm = os.path.join(sDstDirPath, 'TestSet-%d.zip' % (idTestSet,)); + if not os.path.exists(sZipFileNm): + # + # Create zip file with all testset files as members. + # + self.dprint('TestSet %d: Creating %s...' % (idTestSet, sZipFileNm,)); + if not self.fDryRun: + + if not os.path.exists(sDstDirPath): + os.makedirs(sDstDirPath, 0o755); + + utils.noxcptDeleteFile(sZipFileNm + '.tmp'); + oZipFile = zipfile.ZipFile(sZipFileNm + '.tmp', 'w', zipfile.ZIP_DEFLATED, allowZip64 = True); + + for sFile in asFiles: + sSuff = os.path.splitext(sFile)[1]; + if sSuff in [ '.png', '.webm', '.gz', '.bz2', '.zip', '.mov', '.avi', '.mpg', '.gif', '.jpg' ]: + ## @todo Consider storing these files outside the zip if they are a little largish. + self.dprint('TestSet %d: Storing %s...' % (idTestSet, sFile)); + oZipFile.write(sSrcFileBase + sFile, sFile, zipfile.ZIP_STORED); + else: + self.dprint('TestSet %d: Deflating %s...' % (idTestSet, sFile)); + oZipFile.write(sSrcFileBase + sFile, sFile, zipfile.ZIP_DEFLATED); + + oZipFile.close(); + + # + # .zip.tmp -> .zip. + # + utils.noxcptDeleteFile(sZipFileNm); + os.rename(sZipFileNm + '.tmp', sZipFileNm); + + #else: Dry run. + else: + self.dprint('TestSet %d: zip file exists already (%s)' % (idTestSet, sZipFileNm,)); + + # + # Delete the files. + # + fRc = True; + if self.fVerbose: + self.dprint('TestSet %d: deleting file: %s' % (idTestSet, asFiles)); + if not self.fDryRun: + for sFile in asFiles: + if utils.noxcptDeleteFile(sSrcFileBase + sFile) is False: + self.warning('TestSet %d: Failed to delete "%s" (%s)' % (idTestSet, sFile, sSrcFileBase + sFile,)); + fRc = False; + + return fRc; + + + def processDir(self, sCurDir): + """ + Process the given directory (relative to sSrcDir and sDstDir). + Returns success indicator. + """ + if self.fVerbose: + self.dprint('processDir: %s' % (sCurDir,)); + + # + # Sift thought the directory content, collecting subdirectories and + # sort relevant files by test set. + # Generally there will either be subdirs or there will be files. + # + asSubDirs = []; + dTestSets = {}; + sCurPath = os.path.abspath(os.path.join(self.sSrcDir, sCurDir)); + for sFile in os.listdir(sCurPath): + if os.path.isdir(os.path.join(sCurPath, sFile)): + if sFile not in [ '.', '..' ]: + asSubDirs.append(sFile); + elif sFile.startswith('TestSet-'): + # Parse the file name. ASSUMES 'TestSet-%d-filename' format. + iSlash1 = sFile.find('-'); + iSlash2 = sFile.find('-', iSlash1 + 1); + if iSlash2 <= iSlash1: + self.warning('Bad filename (1): "%s"' % (sFile,)); + continue; + + try: idTestSet = int(sFile[(iSlash1 + 1):iSlash2]); + except: + self.warning('Bad filename (2): "%s"' % (sFile,)); + if self.fVerbose: + self.dprint('\n'.join(utils.getXcptInfo(4))); + continue; + + if idTestSet <= 0: + self.warning('Bad filename (3): "%s"' % (sFile,)); + continue; + + if iSlash2 + 2 >= len(sFile): + self.warning('Bad filename (4): "%s"' % (sFile,)); + continue; + sName = sFile[(iSlash2 + 1):]; + + # Add it. + if idTestSet not in dTestSets: + dTestSets[idTestSet] = []; + asTestSet = dTestSets[idTestSet]; + asTestSet.append(sName); + + # + # Test sets. + # + fRc = True; + for idTestSet in dTestSets: + try: + if self._processTestSet(idTestSet, dTestSets[idTestSet], sCurDir) is not True: + fRc = False; + except: + self.warning('TestSet %d: Exception in _processTestSet:\n%s' % (idTestSet, '\n'.join(utils.getXcptInfo()),)); + fRc = False; + + # + # Sub dirs. + # + for sSubDir in asSubDirs: + if self.processDir(os.path.join(sCurDir, sSubDir)) is not True: + fRc = False; + + # + # Try Remove the directory iff it's not '.' and it's been unmodified + # for the last 24h (race protection). + # + if sCurDir != '.': + try: + fpModTime = float(os.path.getmtime(sCurPath)); + if fpModTime + (24*3600) <= time.time(): + if utils.noxcptRmDir(sCurPath) is True: + self.dprint('Removed "%s".' % (sCurPath,)); + except: + pass; + + return fRc; + + @staticmethod + def main(): + """ C-style main(). """ + # + # Parse options. + # + oParser = OptionParser(); + oParser.add_option('-v', '--verbose', dest = 'fVerbose', action = 'store_true', default = False, + help = 'Verbose output.'); + oParser.add_option('-q', '--quiet', dest = 'fVerbose', action = 'store_false', default = False, + help = 'Quiet operation.'); + oParser.add_option('-d', '--dry-run', dest = 'fDryRun', action = 'store_true', default = False, + help = 'Dry run, do not make any changes.'); + (oOptions, asArgs) = oParser.parse_args() + if asArgs != []: + oParser.print_help(); + return 1; + + # + # Do the work. + # + oBatchJob = FileArchiverBatchJob(oOptions); + fRc = oBatchJob.processDir('.'); + return 0 if fRc is True else 1; + +if __name__ == '__main__': + sys.exit(FileArchiverBatchJob.main()); + diff --git a/src/VBox/ValidationKit/testmanager/batch/regen_sched_queues.py b/src/VBox/ValidationKit/testmanager/batch/regen_sched_queues.py new file mode 100755 index 00000000..542854bd --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/batch/regen_sched_queues.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# $Id: regen_sched_queues.py $ +# pylint: disable=C0301 + +""" +Interface used by the admin to regenerate scheduling queues. +""" + +from __future__ import print_function; + +__copyright__ = \ +""" +Copyright (C) 2012-2019 Oracle Corporation + +This file is part of VirtualBox Open Source Edition (OSE), as +available from http://www.virtualbox.org. This file is free software; +you can redistribute it and/or modify it under the terms of the GNU +General Public License (GPL) as published by the Free Software +Foundation, in version 2 as it comes in the "COPYING" file of the +VirtualBox OSE distribution. VirtualBox OSE is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY of any kind. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL) only, as it comes in the "COPYING.CDDL" file of the +VirtualBox OSE distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. +""" +__version__ = "$Revision: 127855 $" + +# Standard python imports +import sys; +import os; +from optparse import OptionParser; # pylint: disable=deprecated-module + +# Add Test Manager's modules path +g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))); +sys.path.append(g_ksTestManagerDir); + +# Test Manager imports +from testmanager.core.db import TMDatabaseConnection; +from testmanager.core.schedulerbase import SchedulerBase; +from testmanager.core.schedgroup import SchedGroupLogic; + + + +class RegenSchedQueues(object): # pylint: disable=R0903 + """ + Regenerates all the scheduling queues. + """ + + def __init__(self): + """ + Parse command line. + """ + + oParser = OptionParser(); + oParser.add_option('-q', '--quiet', dest = 'fQuiet', action = 'store_true', default = False, + help = 'Quiet execution'); + oParser.add_option('-u', '--uid', dest = 'uid', action = 'store', type = 'int', default = 1, + help = 'User ID to accredit with this job'); + oParser.add_option('--profile', dest = 'fProfile', action = 'store_true', default = False, + help = 'User ID to accredit with this job'); + + (self.oConfig, _) = oParser.parse_args(); + + + def doIt(self): + """ + Does the job. + """ + oDb = TMDatabaseConnection(); + + aoGroups = SchedGroupLogic(oDb).getAll(); + iRc = 0; + for oGroup in aoGroups: + if not self.oConfig.fQuiet: + print('%s (ID %#d):' % (oGroup.sName, oGroup.idSchedGroup,)); + try: + (aoErrors, asMessages) = SchedulerBase.recreateQueue(oDb, self.oConfig.uid, oGroup.idSchedGroup, 2); + except Exception as oXcpt: + oDb.rollback(); + print(' !!Hit exception processing "%s": %s' % (oGroup.sName, oXcpt,)); + else: + if not aoErrors: + if not self.oConfig.fQuiet: + print(' Successfully regenerated.'); + else: + iRc = 1; + print(' %d errors:' % (len(aoErrors,))); + for oError in aoErrors: + if oError[1] is None: + print(' !!%s' % (oError[0],)); + else: + print(' !!%s (%s)' % (oError[0], oError[1])); + if asMessages and not self.oConfig.fQuiet: + print(' %d messages:' % (len(asMessages),)); + for sMsg in asMessages: + print(' ##%s' % (sMsg,)); + return iRc; + + @staticmethod + def main(): + """ Main function. """ + oMain = RegenSchedQueues(); + if oMain.oConfig.fProfile is not True: + iRc = oMain.doIt(); + else: + import cProfile; + oProfiler = cProfile.Profile(); + iRc = oProfiler.runcall(oMain.doIt); + oProfiler.print_stats(sort = 'time'); + oProfiler = None; + return iRc; + +if __name__ == '__main__': + sys.exit(RegenSchedQueues().main()); + diff --git a/src/VBox/ValidationKit/testmanager/batch/vcs_import.py b/src/VBox/ValidationKit/testmanager/batch/vcs_import.py new file mode 100755 index 00000000..c840bf3e --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/batch/vcs_import.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# $Id: vcs_import.py $ +# pylint: disable=C0301 + +""" +Cron job for importing revision history for a repository. +""" + +from __future__ import print_function; + +__copyright__ = \ +""" +Copyright (C) 2012-2019 Oracle Corporation + +This file is part of VirtualBox Open Source Edition (OSE), as +available from http://www.virtualbox.org. This file is free software; +you can redistribute it and/or modify it under the terms of the GNU +General Public License (GPL) as published by the Free Software +Foundation, in version 2 as it comes in the "COPYING" file of the +VirtualBox OSE distribution. VirtualBox OSE is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY of any kind. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL) only, as it comes in the "COPYING.CDDL" file of the +VirtualBox OSE distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. +""" +__version__ = "$Revision: 127855 $" + +# Standard python imports +import sys; +import os; +from optparse import OptionParser; # pylint: disable=deprecated-module +import xml.etree.ElementTree as ET; + +# Add Test Manager's modules path +g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))); +sys.path.append(g_ksTestManagerDir); + +# Test Manager imports +from testmanager.core.db import TMDatabaseConnection; +from testmanager.core.vcsrevisions import VcsRevisionData, VcsRevisionLogic; +from common import utils; + +class VcsImport(object): # pylint: disable=R0903 + """ + Imports revision history from a VSC into the Test Manager database. + """ + + def __init__(self): + """ + Parse command line. + """ + + oParser = OptionParser() + oParser.add_option('-e', '--extra-option', dest = 'asExtraOptions', action = 'append', + help = 'Adds a extra option to the command retrieving the log.'); + oParser.add_option('-f', '--full', dest = 'fFull', action = 'store_true', + help = 'Full revision history import.'); + oParser.add_option('-q', '--quiet', dest = 'fQuiet', action = 'store_true', + help = 'Quiet execution'); + oParser.add_option('-R', '--repository', dest = 'sRepository', metavar = '<repository>', + help = 'Version control repository name.'); + oParser.add_option('-s', '--start-revision', dest = 'iStartRevision', metavar = 'start-revision', + type = "int", default = 0, + help = 'The revision to start at when doing a full import.'); + oParser.add_option('-t', '--type', dest = 'sType', metavar = '<type>', + help = 'The VCS type (default: svn)', choices = [ 'svn', ], default = 'svn'); + oParser.add_option('-u', '--url', dest = 'sUrl', metavar = '<url>', + help = 'The VCS URL'); + + (self.oConfig, _) = oParser.parse_args(); + + # Check command line + asMissing = []; + if self.oConfig.sUrl is None: asMissing.append('--url'); + if self.oConfig.sRepository is None: asMissing.append('--repository'); + if asMissing: + sys.stderr.write('syntax error: Missing: %s\n' % (asMissing,)); + sys.exit(1); + + assert self.oConfig.sType == 'svn'; + + def main(self): + """ + Main function. + """ + oDb = TMDatabaseConnection(); + oLogic = VcsRevisionLogic(oDb); + + # Where to start. + iStartRev = 0; + if not self.oConfig.fFull: + iStartRev = oLogic.getLastRevision(self.oConfig.sRepository); + if iStartRev == 0: + iStartRev = self.oConfig.iStartRevision; + + # Construct a command line. + os.environ['LC_ALL'] = 'en_US.utf-8'; + asArgs = [ + 'svn', + 'log', + '--xml', + '--revision', str(iStartRev) + ':HEAD', + ]; + if self.oConfig.asExtraOptions is not None: + asArgs.extend(self.oConfig.asExtraOptions); + asArgs.append(self.oConfig.sUrl); + if not self.oConfig.fQuiet: + print('Executing: %s' % (asArgs,)); + sLogXml = utils.processOutputChecked(asArgs); + + # Parse the XML and add the entries to the database. + oParser = ET.XMLParser(target = ET.TreeBuilder(), encoding = 'utf-8'); + oParser.feed(sLogXml.encode('utf-8')); # does its own decoding and processOutputChecked always gives us decoded utf-8 now. + oRoot = oParser.close(); + + for oLogEntry in oRoot.findall('logentry'): + iRevision = int(oLogEntry.get('revision')); + sAuthor = oLogEntry.findtext('author').strip(); + sDate = oLogEntry.findtext('date').strip(); + sMessage = oLogEntry.findtext('msg', '').strip(); + if sMessage == '': + sMessage = ' '; + elif len(sMessage) > VcsRevisionData.kcchMax_sMessage: + sMessage = sMessage[:VcsRevisionData.kcchMax_sMessage - 4] + ' ...'; + if not self.oConfig.fQuiet: + utils.printOut(u'sDate=%s iRev=%u sAuthor=%s sMsg[%s]=%s' + % (sDate, iRevision, sAuthor, type(sMessage).__name__, sMessage)); + oData = VcsRevisionData().initFromValues(self.oConfig.sRepository, iRevision, sDate, sAuthor, sMessage); + oLogic.addVcsRevision(oData); + oDb.commit(); + + oDb.close(); + return 0; + +if __name__ == '__main__': + sys.exit(VcsImport().main()); + diff --git a/src/VBox/ValidationKit/testmanager/batch/virtual_test_sheriff.py b/src/VBox/ValidationKit/testmanager/batch/virtual_test_sheriff.py new file mode 100755 index 00000000..3f8f2ee3 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/batch/virtual_test_sheriff.py @@ -0,0 +1,1466 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# $Id: virtual_test_sheriff.py $ +# pylint: disable=C0301 + +""" +Virtual Test Sheriff. + +Duties: + - Try to a assign failure reasons to recently failed tests. + - Reboot or disable bad test boxes. + +""" + +from __future__ import print_function; + +__copyright__ = \ +""" +Copyright (C) 2012-2019 Oracle Corporation + +This file is part of VirtualBox Open Source Edition (OSE), as +available from http://www.virtualbox.org. This file is free software; +you can redistribute it and/or modify it under the terms of the GNU +General Public License (GPL) as published by the Free Software +Foundation, in version 2 as it comes in the "COPYING" file of the +VirtualBox OSE distribution. VirtualBox OSE is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY of any kind. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL) only, as it comes in the "COPYING.CDDL" file of the +VirtualBox OSE distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. +""" +__version__ = "$Revision: 127929 $" + + +# Standard python imports +import sys; +import os; +import hashlib; +import subprocess; +import smtplib +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from email.utils import COMMASPACE + +if sys.version_info[0] >= 3: + from io import StringIO as StringIO; # pylint: disable=import-error,no-name-in-module +else: + from StringIO import StringIO as StringIO; # pylint: disable=import-error,no-name-in-module +from optparse import OptionParser; # pylint: disable=deprecated-module +from PIL import Image; # pylint: disable=import-error + +# Add Test Manager's modules path +g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))); +sys.path.append(g_ksTestManagerDir); + +# Test Manager imports +from testmanager.core.db import TMDatabaseConnection; +from testmanager.core.build import BuildDataEx; +from testmanager.core.failurereason import FailureReasonLogic; +from testmanager.core.testbox import TestBoxLogic, TestBoxData; +from testmanager.core.testcase import TestCaseDataEx; +from testmanager.core.testgroup import TestGroupData; +from testmanager.core.testset import TestSetLogic, TestSetData; +from testmanager.core.testresults import TestResultLogic, TestResultFileData; +from testmanager.core.testresultfailures import TestResultFailureLogic, TestResultFailureData; +from testmanager.core.useraccount import UserAccountLogic; +from testmanager.config import g_ksSmtpHost, g_kcSmtpPort, g_ksAlertFrom, \ + g_ksAlertSubject, g_asAlertList, g_ksLomPassword; + +# Python 3 hacks: +if sys.version_info[0] >= 3: + xrange = range; # pylint: disable=redefined-builtin,invalid-name + + +class VirtualTestSheriffCaseFile(object): + """ + A failure investigation case file. + + """ + + + ## Max log file we'll read into memory. (256 MB) + kcbMaxLogRead = 0x10000000; + + def __init__(self, oSheriff, oTestSet, oTree, oBuild, oTestBox, oTestGroup, oTestCase): + self.oSheriff = oSheriff; + self.oTestSet = oTestSet; # TestSetData + self.oTree = oTree; # TestResultDataEx + self.oBuild = oBuild; # BuildDataEx + self.oTestBox = oTestBox; # TestBoxData + self.oTestGroup = oTestGroup; # TestGroupData + self.oTestCase = oTestCase; # TestCaseDataEx + self.sMainLog = ''; # The main log file. Empty string if not accessible. + + # Generate a case file name. + self.sName = '#%u: %s' % (self.oTestSet.idTestSet, self.oTestCase.sName,) + self.sLongName = '#%u: "%s" on "%s" running %s %s (%s), "%s" by %s, using %s %s %s r%u' \ + % ( self.oTestSet.idTestSet, + self.oTestCase.sName, + self.oTestBox.sName, + self.oTestBox.sOs, + self.oTestBox.sOsVersion, + self.oTestBox.sCpuArch, + self.oTestBox.sCpuName, + self.oTestBox.sCpuVendor, + self.oBuild.oCat.sProduct, + self.oBuild.oCat.sBranch, + self.oBuild.oCat.sType, + self.oBuild.iRevision, ); + + # Investigation notes. + self.tReason = None; # None or one of the ktReason_XXX constants. + self.dReasonForResultId = {}; # Reason assignments indexed by idTestResult. + self.dCommentForResultId = {}; # Comment assignments indexed by idTestResult. + + # + # Reason. + # + + def noteReason(self, tReason): + """ Notes down a possible reason. """ + self.oSheriff.dprint(u'noteReason: %s -> %s' % (self.tReason, tReason,)); + self.tReason = tReason; + return True; + + def noteReasonForId(self, tReason, idTestResult, sComment = None): + """ Notes down a possible reason for a specific test result. """ + self.oSheriff.dprint(u'noteReasonForId: %u: %s -> %s%s' + % (idTestResult, self.dReasonForResultId.get(idTestResult, None), tReason, + (u' (%s)' % (sComment,)) if sComment is not None else '')); + self.dReasonForResultId[idTestResult] = tReason; + if sComment is not None: + self.dCommentForResultId[idTestResult] = sComment; + return True; + + + # + # Test classification. + # + + def isVBoxTest(self): + """ Test classification: VirtualBox (using the build) """ + return self.oBuild.oCat.sProduct.lower() in [ 'virtualbox', 'vbox' ]; + + def isVBoxUnitTest(self): + """ Test case classification: The unit test doing all our testcase/*.cpp stuff. """ + return self.isVBoxTest() \ + and (self.oTestCase.sName.lower() == 'unit tests' or self.oTestCase.sName.lower() == 'misc: unit tests'); + + def isVBoxInstallTest(self): + """ Test case classification: VirtualBox Guest installation test. """ + return self.isVBoxTest() \ + and self.oTestCase.sName.lower().startswith('install:'); + + def isVBoxUSBTest(self): + """ Test case classification: VirtualBox USB test. """ + return self.isVBoxTest() \ + and self.oTestCase.sName.lower().startswith('usb:'); + + def isVBoxStorageTest(self): + """ Test case classification: VirtualBox Storage test. """ + return self.isVBoxTest() \ + and self.oTestCase.sName.lower().startswith('storage:'); + + def isVBoxGAsTest(self): + """ Test case classification: VirtualBox Guest Additions test. """ + return self.isVBoxTest() \ + and self.oTestCase.sName.lower().startswith('ga\'s tests'); + + def isVBoxAPITest(self): + """ Test case classification: VirtualBox API test. """ + return self.isVBoxTest() \ + and self.oTestCase.sName.lower().startswith('api:'); + + def isVBoxBenchmarkTest(self): + """ Test case classification: VirtualBox Benchmark test. """ + return self.isVBoxTest() \ + and self.oTestCase.sName.lower().startswith('benchmark:'); + + def isVBoxSmokeTest(self): + """ Test case classification: Smoke test. """ + return self.isVBoxTest() \ + and self.oTestCase.sName.lower().startswith('smoketest'); + + + # + # Utility methods. + # + + def getMainLog(self): + """ + Tries to read the main log file since this will be the first source of information. + """ + if self.sMainLog: + return self.sMainLog; + (oFile, oSizeOrError, _) = self.oTestSet.openFile('main.log', 'rb'); + if oFile is not None: + try: + self.sMainLog = oFile.read(min(self.kcbMaxLogRead, oSizeOrError)).decode('utf-8', 'replace'); + except Exception as oXcpt: + self.oSheriff.vprint(u'Error reading main log file: %s' % (oXcpt,)) + self.sMainLog = ''; + else: + self.oSheriff.vprint(u'Error opening main log file: %s' % (oSizeOrError,)); + return self.sMainLog; + + def getLogFile(self, oFile): + """ + Tries to read the given file as a utf-8 log file. + oFile is a TestFileDataEx instance. + Returns empty string if problems opening or reading the file. + """ + sContent = ''; + (oFile, oSizeOrError, _) = self.oTestSet.openFile(oFile.sFile, 'rb'); + if oFile is not None: + try: + sContent = oFile.read(min(self.kcbMaxLogRead, oSizeOrError)).decode('utf-8', 'replace'); + except Exception as oXcpt: + self.oSheriff.vprint(u'Error reading the "%s" log file: %s' % (oFile.sFile, oXcpt,)) + else: + self.oSheriff.vprint(u'Error opening the "%s" log file: %s' % (oFile.sFile, oSizeOrError,)); + return sContent; + + def getScreenshotSha256(self, oFile): + """ + Tries to read the given screenshot file, uncompress it, and do SHA-2 + on the raw pixels. + Returns SHA-2 digest string on success, None on failure. + """ + (oImgFile, _, _) = self.oTestSet.openFile(oFile.sFile, 'rb'); + try: + abImageFile = oImgFile.read(); + except Exception as oXcpt: + self.oSheriff.vprint(u'Error reading the "%s" image file: %s' % (oFile.sFile, oXcpt,)) + else: + try: + oImage = Image.open(StringIO(abImageFile)); + except Exception as oXcpt: + self.oSheriff.vprint(u'Error opening the "%s" image bytes using PIL.Image.open: %s' % (oFile.sFile, oXcpt,)) + else: + try: + oHash = hashlib.sha256(); + oHash.update(oImage.tostring()); + except Exception as oXcpt: + self.oSheriff.vprint(u'Error hashing the uncompressed image bytes for "%s": %s' % (oFile.sFile, oXcpt,)) + else: + return oHash.hexdigest(); + return None; + + + + def isSingleTestFailure(self): + """ + Figure out if this is a single test failing or if it's one of the + more complicated ones. + """ + if self.oTree.cErrors == 1: + return True; + if self.oTree.deepCountErrorContributers() <= 1: + return True; + return False; + + + +class VirtualTestSheriff(object): # pylint: disable=R0903 + """ + Add build info into Test Manager database. + """ + + ## The user account for the virtual sheriff. + ksLoginName = 'vsheriff'; + + def __init__(self): + """ + Parse command line. + """ + self.oDb = None; + self.tsNow = None; + self.oTestResultLogic = None; + self.oTestSetLogic = None; + self.oFailureReasonLogic = None; # FailureReasonLogic; + self.oTestResultFailureLogic = None; # TestResultFailureLogic + self.oLogin = None; + self.uidSelf = -1; + self.oLogFile = None; + self.asBsodReasons = []; + self.asUnitTestReasons = []; + + oParser = OptionParser(); + oParser.add_option('--start-hours-ago', dest = 'cStartHoursAgo', metavar = '<hours>', default = 0, type = 'int', + help = 'When to start specified as hours relative to current time. Defauls is right now.', ); + oParser.add_option('--hours-period', dest = 'cHoursBack', metavar = '<period-in-hours>', default = 2, type = 'int', + help = 'Work period specified in hours. Defauls is 2 hours.'); + oParser.add_option('--real-run-back', dest = 'fRealRun', action = 'store_true', default = False, + help = 'Whether to commit the findings to the database. Default is a dry run.'); + oParser.add_option('-q', '--quiet', dest = 'fQuiet', action = 'store_true', default = False, + help = 'Quiet execution'); + oParser.add_option('-l', '--log', dest = 'sLogFile', metavar = '<logfile>', default = None, + help = 'Where to log messages.'); + oParser.add_option('--debug', dest = 'fDebug', action = 'store_true', default = False, + help = 'Enables debug mode.'); + + (self.oConfig, _) = oParser.parse_args(); + + if self.oConfig.sLogFile: + self.oLogFile = open(self.oConfig.sLogFile, "a"); + self.oLogFile.write('VirtualTestSheriff: $Revision: 127929 $ \n'); + + + def eprint(self, sText): + """ + Prints error messages. + Returns 1 (for exit code usage.) + """ + print('error: %s' % (sText,)); + if self.oLogFile is not None: + self.oLogFile.write((u'error: %s\n' % (sText,)).encode('utf-8')); + return 1; + + def dprint(self, sText): + """ + Prints debug info. + """ + if self.oConfig.fDebug: + if not self.oConfig.fQuiet: + print('debug: %s' % (sText, )); + if self.oLogFile is not None: + self.oLogFile.write((u'debug: %s\n' % (sText,)).encode('utf-8')); + return 0; + + def vprint(self, sText): + """ + Prints verbose info. + """ + if not self.oConfig.fQuiet: + print('info: %s' % (sText,)); + if self.oLogFile is not None: + self.oLogFile.write((u'info: %s\n' % (sText,)).encode('utf-8')); + return 0; + + def getFailureReason(self, tReason): + """ Gets the failure reason object for tReason. """ + return self.oFailureReasonLogic.cachedLookupByNameAndCategory(tReason[1], tReason[0]); + + def selfCheck(self): + """ Does some self checks, looking up things we expect to be in the database and such. """ + rcExit = 0; + for sAttr in dir(self.__class__): + if sAttr.startswith('ktReason_'): + tReason = getattr(self.__class__, sAttr); + oFailureReason = self.getFailureReason(tReason); + if oFailureReason is None: + rcExit = self.eprint(u'Failed to find failure reason "%s" in category "%s" in the database!' + % (tReason[1], tReason[0],)); + + # Check the user account as well. + if self.oLogin is None: + oLogin = UserAccountLogic(self.oDb).tryFetchAccountByLoginName(VirtualTestSheriff.ksLoginName); + if oLogin is None: + rcExit = self.eprint(u'Cannot find my user account "%s"!' % (VirtualTestSheriff.ksLoginName,)); + return rcExit; + + def sendEmailAlert(self, uidAuthor, sBodyText): + """ + Sends email alert. + """ + + # Get author email + self.oDb.execute('SELECT sEmail FROM Users WHERE uid=%s', (uidAuthor,)); + sFrom = self.oDb.fetchOne(); + if sFrom is not None: + sFrom = sFrom[0]; + else: + sFrom = g_ksAlertFrom; + + # Gather recipient list. + asEmailList = []; + for sUser in g_asAlertList: + self.oDb.execute('SELECT sEmail FROM Users WHERE sUsername=%s', (sUser,)); + sEmail = self.oDb.fetchOne(); + if sEmail: + asEmailList.append(sEmail[0]); + if not asEmailList: + return self.eprint('No email addresses to send alter to!'); + + # Compose the message. + oMsg = MIMEMultipart(); + oMsg['From'] = sFrom; + oMsg['To'] = COMMASPACE.join(asEmailList); + oMsg['Subject'] = g_ksAlertSubject; + oMsg.attach(MIMEText(sBodyText, 'plain')) + + # Try send it. + try: + oSMTP = smtplib.SMTP(g_ksSmtpHost, g_kcSmtpPort); + oSMTP.sendmail(sFrom, asEmailList, oMsg.as_string()) + oSMTP.quit() + except smtplib.SMTPException as oXcpt: + return self.eprint('Failed to send mail: %s' % (oXcpt,)); + + return 0; + + def badTestBoxManagement(self): + """ + Looks for bad test boxes and first tries once to reboot them then disables them. + """ + rcExit = 0; + + # + # We skip this entirely if we're running in the past and not in harmless debug mode. + # + if self.oConfig.cStartHoursAgo != 0 \ + and (not self.oConfig.fDebug or self.oConfig.fRealRun): + return rcExit; + tsNow = self.tsNow if self.oConfig.fDebug else None; + cHoursBack = self.oConfig.cHoursBack if self.oConfig.fDebug else 2; + oTestBoxLogic = TestBoxLogic(self.oDb); + + # + # Generate a list of failures reasons we consider bad-testbox behavior. + # + aidFailureReasons = [ + self.getFailureReason(self.ktReason_Host_DriverNotUnloading).idFailureReason, + self.getFailureReason(self.ktReason_Host_DriverNotCompilable).idFailureReason, + self.getFailureReason(self.ktReason_Host_InstallationFailed).idFailureReason, + ]; + + # + # Get list of bad test boxes for given period and check them out individually. + # + aidBadTestBoxes = self.oTestSetLogic.fetchBadTestBoxIds(cHoursBack = cHoursBack, tsNow = tsNow, + aidFailureReasons = aidFailureReasons); + for idTestBox in aidBadTestBoxes: + # Skip if the testbox is already disabled or has a pending reboot command. + try: + oTestBox = TestBoxData().initFromDbWithId(self.oDb, idTestBox); + except Exception as oXcpt: + rcExit = self.eprint('Failed to get data for test box #%u in badTestBoxManagement: %s' % (idTestBox, oXcpt,)); + continue; + if not oTestBox.fEnabled: + self.dprint(u'badTestBoxManagement: Skipping test box #%u (%s) as it has been disabled already.' + % ( idTestBox, oTestBox.sName, )); + continue; + if oTestBox.enmPendingCmd != TestBoxData.ksTestBoxCmd_None: + self.dprint(u'badTestBoxManagement: Skipping test box #%u (%s) as it has a command pending: %s' + % ( idTestBox, oTestBox.sName, oTestBox.enmPendingCmd)); + continue; + + # Get the most recent testsets for this box (descending on tsDone) and see how bad it is. + aoSets = self.oTestSetLogic.fetchSetsForTestBox(idTestBox, cHoursBack = cHoursBack, tsNow = tsNow); + cOkay = 0; + cBad = 0; + iFirstOkay = len(aoSets); + for iSet, oSet in enumerate(aoSets): + if oSet.enmStatus == TestSetData.ksTestStatus_BadTestBox: + cBad += 1; + else: + # Check for bad failure reasons. + oFailure = None; + if oSet.enmStatus in TestSetData.kasBadTestStatuses: + oFailure = self.oTestResultFailureLogic.getById(oSet.idTestResult); + if oFailure is not None and oFailure.idFailureReason in aidFailureReasons: + cBad += 1; + else: + # This is an okay test result then. + ## @todo maybe check the elapsed time here, it could still be a bad run? + cOkay += 1; + if iFirstOkay > iSet: + iFirstOkay = iSet; + if iSet > 10: + break; + + # We react if there are two or more bad-testbox statuses at the head of the + # history and at least three in the last 10 results. + if iFirstOkay >= 2 and cBad > 2: + # Frank: For now don't reboot boxes automatically + if True or oTestBoxLogic.hasTestBoxRecentlyBeenRebooted(idTestBox, cHoursBack = cHoursBack, tsNow = tsNow): + self.vprint(u'Disabling testbox #%u (%s) - iFirstOkay=%u cBad=%u cOkay=%u' + % ( idTestBox, oTestBox.sName, iFirstOkay, cBad, cOkay)); + if self.oConfig.fRealRun is True: + try: + oTestBoxLogic.disableTestBox(idTestBox, self.uidSelf, fCommit = True, + sComment = 'Automatically disabled (iFirstOkay=%u cBad=%u cOkay=%u)' + % (iFirstOkay, cBad, cOkay),); + except Exception as oXcpt: + rcExit = self.eprint(u'Error disabling testbox #%u (%u): %s\n' % (idTestBox, oTestBox.sName, oXcpt,)); + else: + self.vprint(u'Rebooting testbox #%u (%s) - iFirstOkay=%u cBad=%u cOkay=%u' + % ( idTestBox, oTestBox.sName, iFirstOkay, cBad, cOkay)); + if self.oConfig.fRealRun is True: + try: + oTestBoxLogic.rebootTestBox(idTestBox, self.uidSelf, fCommit = True, + sComment = 'Automatically rebooted (iFirstOkay=%u cBad=%u cOkay=%u)' + % (iFirstOkay, cBad, cOkay),); + except Exception as oXcpt: + rcExit = self.eprint(u'Error rebooting testbox #%u (%s): %s\n' % (idTestBox, oTestBox.sName, oXcpt,)); + else: + self.dprint(u'badTestBoxManagement: #%u (%s) looks ok: iFirstOkay=%u cBad=%u cOkay=%u' + % ( idTestBox, oTestBox.sName, iFirstOkay, cBad, cOkay)); + # + # Reset hanged testboxes + # + cStatusTimeoutMins = 10; + + self.oDb.execute('SELECT idTestBox FROM TestBoxStatuses WHERE tsUpdated < (CURRENT_TIMESTAMP - interval \'%s minutes\')', (cStatusTimeoutMins,)); + for idTestBox in self.oDb.fetchAll(): + idTestBox = idTestBox[0]; + try: + oTestBox = TestBoxData().initFromDbWithId(self.oDb, idTestBox); + except Exception as oXcpt: + rcExit = self.eprint('Failed to get data for test box #%u in badTestBoxManagement: %s' % (idTestBox, oXcpt,)); + continue; + # Skip if the testbox is already disabled, already reset or there's no iLOM + if not oTestBox.fEnabled or oTestBox.ipLom is None or oTestBox.sComment is not None and oTestBox.sComment.find('Automatically reset') >= 0: + self.dprint(u'badTestBoxManagement: Skipping test box #%u (%s) as it has been disabled already.' + % ( idTestBox, oTestBox.sName, )); + continue; + ## @todo get iLOM credentials from a table? + sCmd = 'sshpass -p%s ssh -oStrictHostKeyChecking=no root@%s show /SP && reset /SYS' % (g_ksLomPassword, oTestBox.ipLom,); + try: + oPs = subprocess.Popen(sCmd, stdout=subprocess.PIPE, shell=True); + sStdout = oPs.communicate()[0]; + iRC = oPs.wait(); + + oTestBox.sComment = 'Automatically reset (iRC=%u sStdout=%s)' % (iRC, sStdout,); + oTestBoxLogic.editEntry(oTestBox, self.uidSelf, fCommit = True); + + sComment = u'Reset testbox #%u (%s) - iRC=%u sStduot=%s' % ( idTestBox, oTestBox.sName, iRC, sStdout); + self.vprint(sComment); + self.sendEmailAlert(self.uidSelf, sComment); + + except Exception as oXcpt: + rcExit = self.eprint(u'Error reseting testbox #%u (%s): %s\n' % (idTestBox, oTestBox.sName, oXcpt,)); + + return rcExit; + + + ## @name Failure reasons we know. + ## @{ + ktReason_BSOD_Recovery = ( 'BSOD', 'Recovery' ); + ktReason_BSOD_Automatic_Repair = ( 'BSOD', 'Automatic Repair' ); + ktReason_BSOD_0000007F = ( 'BSOD', '0x0000007F' ); + ktReason_BSOD_000000D1 = ( 'BSOD', '0x000000D1' ); + ktReason_BSOD_C0000225 = ( 'BSOD', '0xC0000225 (boot)' ); + ktReason_Guru_Generic = ( 'Guru Meditations', 'Generic Guru Meditation' ); + ktReason_Guru_VERR_IEM_INSTR_NOT_IMPLEMENTED = ( 'Guru Meditations', 'VERR_IEM_INSTR_NOT_IMPLEMENTED' ); + ktReason_Guru_VERR_IEM_ASPECT_NOT_IMPLEMENTED = ( 'Guru Meditations', 'VERR_IEM_ASPECT_NOT_IMPLEMENTED' ); + ktReason_Guru_VERR_TRPM_DONT_PANIC = ( 'Guru Meditations', 'VERR_TRPM_DONT_PANIC' ); + ktReason_Guru_VERR_PGM_PHYS_PAGE_RESERVED = ( 'Guru Meditations', 'VERR_PGM_PHYS_PAGE_RESERVED' ); + ktReason_Guru_VERR_VMX_INVALID_GUEST_STATE = ( 'Guru Meditations', 'VERR_VMX_INVALID_GUEST_STATE' ); + ktReason_Guru_VINF_EM_TRIPLE_FAULT = ( 'Guru Meditations', 'VINF_EM_TRIPLE_FAULT' ); + ktReason_Host_HostMemoryLow = ( 'Host', 'HostMemoryLow' ); + ktReason_Host_DriverNotLoaded = ( 'Host', 'Driver not loaded' ); + ktReason_Host_DriverNotUnloading = ( 'Host', 'Driver not unloading' ); + ktReason_Host_DriverNotCompilable = ( 'Host', 'Driver not compilable' ); + ktReason_Host_InstallationFailed = ( 'Host', 'Installation failed' ); + ktReason_Host_NotSignedWithBuildCert = ( 'Host', 'Not signed with build cert' ); + ktReason_Host_DoubleFreeHeap = ( 'Host', 'Double free or corruption' ); + ktReason_Host_LeftoverService = ( 'Host', 'Leftover service' ); + ktReason_Host_Reboot_OSX_Watchdog_Timeout = ( 'Host Reboot', 'OSX Watchdog Timeout' ); + ktReason_Host_Modprobe_Failed = ( 'Host', 'Modprobe failed' ); + ktReason_Host_Install_Hang = ( 'Host', 'Install hang' ); + ktReason_Host_NetworkMisconfiguration = ( 'Host', 'Network misconfiguration' ); + ktReason_Networking_Nonexistent_host_nic = ( 'Networking', 'Nonexistent host networking interface' ); + ktReason_OSInstall_GRUB_hang = ( 'O/S Install', 'GRUB hang' ); + ktReason_OSInstall_Udev_hang = ( 'O/S Install', 'udev hang' ); + ktReason_OSInstall_Sata_no_BM = ( 'O/S Install', 'SATA busmaster bit not set' ); + ktReason_Panic_BootManagerC000000F = ( 'Panic', 'Hardware Changed' ); + ktReason_BootManager_Image_corrupt = ( 'Unknown', 'BOOTMGR Image corrupt' ); + ktReason_Panic_MP_BIOS_IO_APIC = ( 'Panic', 'MP-BIOS/IO-APIC' ); + ktReason_Panic_HugeMemory = ( 'Panic', 'Huge memory assertion' ); + ktReason_Panic_IOAPICDoesntWork = ( 'Panic', 'IO-APIC and timer does not work' ); + ktReason_Panic_TxUnitHang = ( 'Panic', 'Tx Unit Hang' ); + ktReason_XPCOM_Exit_Minus_11 = ( 'API / (XP)COM', 'exit -11' ); + ktReason_XPCOM_VBoxSVC_Hang = ( 'API / (XP)COM', 'VBoxSVC hang' ); + ktReason_XPCOM_VBoxSVC_Hang_Plus_Heap_Corruption = ( 'API / (XP)COM', 'VBoxSVC hang + heap corruption' ); + ktReason_XPCOM_NS_ERROR_CALL_FAILED = ( 'API / (XP)COM', 'NS_ERROR_CALL_FAILED' ); + ktReason_Unknown_Heap_Corruption = ( 'Unknown', 'Heap corruption' ); + ktReason_Unknown_Reboot_Loop = ( 'Unknown', 'Reboot loop' ); + ktReason_Unknown_File_Not_Found = ( 'Unknown', 'File not found' ); + ktReason_Unknown_VM_Crash = ( 'Unknown', 'VM crash' ); + ktReason_Unknown_HalReturnToFirmware = ( 'Unknown', 'HalReturnToFirmware' ); + ktReason_VMM_kvm_lock_spinning = ( 'VMM', 'kvm_lock_spinning' ); + ktReason_Ignore_Buggy_Test_Driver = ( 'Ignore', 'Buggy test driver' ); + ktReason_Ignore_Stale_Files = ( 'Ignore', 'Stale files' ); + ktReason_Buggy_Build_Broken_Build = ( 'Broken Build', 'Buggy build' ); + ktReason_Unknown_VM_Start_Error = ( 'Unknown', 'VM Start Error' ); + ktReason_Unknown_VM_Runtime_Error = ( 'Unknown', 'VM Runtime Error' ); + ktReason_GuestBug_CompizVBoxQt = ( 'Guest Bug', 'Compiz + VirtualBox Qt GUI crash' ); + ## @} + + ## BSOD category. + ksBsodCategory = 'BSOD'; + ## Special reason indicating that the flesh and blood sheriff has work to do. + ksBsodAddNew = 'Add new BSOD'; + + ## Unit test category. + ksUnitTestCategory = 'Unit'; + ## Special reason indicating that the flesh and blood sheriff has work to do. + ksUnitTestAddNew = 'Add new'; + + ## Used for indica that we shouldn't report anything for this test result ID and + ## consider promoting the previous error to test set level if it's the only one. + ktHarmless = ( 'Probably', 'Caused by previous error' ); + + + def caseClosed(self, oCaseFile): + """ + Reports the findings in the case and closes it. + """ + # + # Log it and create a dReasonForReasultId we can use below. + # + dCommentForResultId = oCaseFile.dCommentForResultId; + if oCaseFile.dReasonForResultId: + # Must weed out ktHarmless. + dReasonForResultId = {}; + for idKey, tReason in oCaseFile.dReasonForResultId.items(): + if tReason is not self.ktHarmless: + dReasonForResultId[idKey] = tReason; + if not dReasonForResultId: + self.vprint(u'TODO: Closing %s without a real reason, only %s.' + % (oCaseFile.sName, oCaseFile.dReasonForResultId)); + return False; + + # Try promote to single reason. + atValues = dReasonForResultId.values(); + fSingleReason = True; + if len(dReasonForResultId) == 1 and dReasonForResultId.keys()[0] != oCaseFile.oTestSet.idTestResult: + self.dprint(u'Promoting single reason to whole set: %s' % (atValues[0],)); + elif len(dReasonForResultId) > 1 and len(atValues) == atValues.count(atValues[0]): + self.dprint(u'Merged %d reasons to a single one: %s' % (len(atValues), atValues[0])); + else: + fSingleReason = False; + if fSingleReason: + dReasonForResultId = { oCaseFile.oTestSet.idTestResult: atValues[0], }; + if dCommentForResultId: + dCommentForResultId = { oCaseFile.oTestSet.idTestResult: dCommentForResultId.values()[0], }; + elif oCaseFile.tReason is not None: + dReasonForResultId = { oCaseFile.oTestSet.idTestResult: oCaseFile.tReason, }; + else: + self.vprint(u'Closing %s without a reason - this should not happen!' % (oCaseFile.sName,)); + return False; + + self.vprint(u'Closing %s with following reason%s: %s' + % ( oCaseFile.sName, 's' if dReasonForResultId > 0 else '', dReasonForResultId, )); + + # + # Add the test failure reason record(s). + # + for idTestResult, tReason in dReasonForResultId.items(): + oFailureReason = self.getFailureReason(tReason); + if oFailureReason is not None: + sComment = 'Set by $Revision: 127929 $' # Handy for reverting later. + if idTestResult in dCommentForResultId: + sComment += ': ' + dCommentForResultId[idTestResult]; + + oAdd = TestResultFailureData(); + oAdd.initFromValues(idTestResult = idTestResult, + idFailureReason = oFailureReason.idFailureReason, + uidAuthor = self.uidSelf, + idTestSet = oCaseFile.oTestSet.idTestSet, + sComment = sComment,); + if self.oConfig.fRealRun: + try: + self.oTestResultFailureLogic.addEntry(oAdd, self.uidSelf, fCommit = True); + except Exception as oXcpt: + self.eprint(u'caseClosed: Exception "%s" while adding reason %s for %s' + % (oXcpt, oAdd, oCaseFile.sLongName,)); + else: + self.eprint(u'caseClosed: Cannot locate failure reason: %s / %s' % ( tReason[0], tReason[1],)); + return True; + + # + # Tools for assiting log parsing. + # + + @staticmethod + def matchFollowedByLines(sStr, off, asFollowingLines): + """ Worker for isThisFollowedByTheseLines. """ + + # Advance off to the end of the line. + off = sStr.find('\n', off); + if off < 0: + return False; + off += 1; + + # Match each string with the subsequent lines. + for iLine, sLine in enumerate(asFollowingLines): + offEnd = sStr.find('\n', off); + if offEnd < 0: + return iLine + 1 == len(asFollowingLines) and sStr.find(sLine, off) < 0; + if sLine and sStr.find(sLine, off, offEnd) < 0: + return False; + + # next line. + off = offEnd + 1; + + return True; + + @staticmethod + def isThisFollowedByTheseLines(sStr, sFirst, asFollowingLines): + """ + Looks for a line contining sFirst which is then followed by lines + with the strings in asFollowingLines. (No newline chars anywhere!) + Returns True / False. + """ + off = sStr.find(sFirst, 0); + while off >= 0: + if VirtualTestSheriff.matchFollowedByLines(sStr, off, asFollowingLines): + return True; + off = sStr.find(sFirst, off + 1); + return False; + + @staticmethod + def findAndReturnRestOfLine(sHaystack, sNeedle): + """ + Looks for sNeedle in sHaystack. + Returns The text following the needle up to the end of the line. + Returns None if not found. + """ + if sHaystack is None: + return None; + off = sHaystack.find(sNeedle); + if off < 0: + return None; + off += len(sNeedle) + offEol = sHaystack.find('\n', off); + if offEol < 0: + offEol = len(sHaystack); + return sHaystack[off:offEol] + + @staticmethod + def findInAnyAndReturnRestOfLine(asHaystacks, sNeedle): + """ + Looks for sNeedle in zeroe or more haystacks (asHaystack). + Returns The text following the first needed found up to the end of the line. + Returns None if not found. + """ + for sHaystack in asHaystacks: + sRet = VirtualTestSheriff.findAndReturnRestOfLine(sHaystack, sNeedle); + if sRet is not None: + return sRet; + return None; + + + # + # The investigative units. + # + + katSimpleInstallUninstallMainLogReasons = [ + # ( Whether to stop on hit, reason tuple, needle text. ) + ( False, ktReason_Host_LeftoverService, + 'SERVICE_NAME: vbox' ), + ]; + + kdatSimpleInstallUninstallMainLogReasonsPerOs = { + 'darwin': [ + # ( Whether to stop on hit, reason tuple, needle text. ) + ( True, ktReason_Host_DriverNotUnloading, + 'Can\'t remove kext org.virtualbox.kext.VBoxDrv; services failed to terminate - 0xe00002c7' ), + ], + 'linux': [ + # ( Whether to stop on hit, reason tuple, needle text. ) + ( True, ktReason_Host_DriverNotCompilable, + 'This system is not currently set up to build kernel modules' ), + ( True, ktReason_Host_DriverNotCompilable, + 'This system is currently not set up to build kernel modules' ), + ( True, ktReason_Host_InstallationFailed, + 'vboxdrv.sh: failed: Look at /var/log/vbox-install.log to find out what went wrong.' ), + ( True, ktReason_Host_DriverNotUnloading, + 'Cannot unload module vboxdrv'), + ], + }; + + + def investigateInstallUninstallFailure(self, oCaseFile, oFailedResult, sResultLog, fInstall): + """ + Investigates an install or uninstall failure. + + We lump the two together since the installation typically also performs + an uninstall first and will be seeing similar issues to the uninstall. + """ + + if fInstall and oFailedResult.enmStatus == TestSetData.ksTestStatus_TimedOut: + oCaseFile.noteReasonForId(self.ktReason_Host_Install_Hang, oFailedResult.idTestResult) + return True; + + atSimple = self.katSimpleInstallUninstallMainLogReasons; + if oCaseFile.oTestBox.sOs in self.kdatSimpleInstallUninstallMainLogReasonsPerOs: + atSimple = self.kdatSimpleInstallUninstallMainLogReasonsPerOs[oCaseFile.oTestBox.sOs] + atSimple; + + fFoundSomething = False; + for fStopOnHit, tReason, sNeedle in atSimple: + if sResultLog.find(sNeedle) > 0: + oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult); + if fStopOnHit: + return True; + fFoundSomething = True; + + return fFoundSomething if fFoundSomething else None; + + + def investigateBadTestBox(self, oCaseFile): + """ + Checks out bad-testbox statuses. + """ + _ = oCaseFile; + return False; + + + def investigateVBoxUnitTest(self, oCaseFile): + """ + Checks out a VBox unittest problem. + """ + + # + # Process simple test case failures first, using their name as reason. + # We do the reason management just like for BSODs. + # + cRelevantOnes = 0; + sMainLog = oCaseFile.getMainLog(); + aoFailedResults = oCaseFile.oTree.getListOfFailures(); + for oFailedResult in aoFailedResults: + if oFailedResult is oCaseFile.oTree: + self.vprint('TODO: toplevel failure'); + cRelevantOnes += 1 + + elif oFailedResult.sName == 'Installing VirtualBox': + sResultLog = TestSetData.extractLogSectionElapsed(sMainLog, oFailedResult.tsCreated, oFailedResult.tsElapsed); + self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = True) + cRelevantOnes += 1 + + elif oFailedResult.sName == 'Uninstalling VirtualBox': + sResultLog = TestSetData.extractLogSectionElapsed(sMainLog, oFailedResult.tsCreated, oFailedResult.tsElapsed); + self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = False) + cRelevantOnes += 1 + + elif oFailedResult.oParent is not None: + # Get the 2nd level node because that's where we'll find the unit test name. + while oFailedResult.oParent.oParent is not None: + oFailedResult = oFailedResult.oParent; + + # Only report a failure once. + if oFailedResult.idTestResult not in oCaseFile.dReasonForResultId: + sKey = oFailedResult.sName; + if sKey.startswith('testcase/'): + sKey = sKey[9:]; + if sKey in self.asUnitTestReasons: + tReason = ( self.ksUnitTestCategory, sKey ); + oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult); + else: + self.dprint(u'Unit test failure "%s" not found in %s;' % (sKey, self.asUnitTestReasons)); + tReason = ( self.ksUnitTestCategory, self.ksUnitTestAddNew ); + oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult, sComment = sKey); + cRelevantOnes += 1 + else: + self.vprint(u'Internal error: expected oParent to NOT be None for %s' % (oFailedResult,)); + + # + # If we've caught all the relevant ones by now, report the result. + # + if len(oCaseFile.dReasonForResultId) >= cRelevantOnes: + return self.caseClosed(oCaseFile); + return False; + + def extractGuestCpuStack(self, sInfoText): + """ + Extracts the guest CPU stacks from the input file. + + Returns a dictionary keyed by the CPU number, value being a list of + raw stack lines (no header). + Returns empty dictionary if no stacks where found. + """ + dRet = {}; + off = 0; + while True: + # Find the stack. + offStart = sInfoText.find('=== start guest stack VCPU ', off); + if offStart < 0: + break; + offEnd = sInfoText.find('=== end guest stack', offStart + 20); + if offEnd >= 0: + offEnd += 3; + else: + offEnd = sInfoText.find('=== start guest stack VCPU', offStart + 20); + if offEnd < 0: + offEnd = len(sInfoText); + + sStack = sInfoText[offStart : offEnd]; + sStack = sStack.replace('\r',''); # paranoia + asLines = sStack.split('\n'); + + # Figure the CPU. + asWords = asLines[0].split(); + if len(asWords) < 6 or not asWords[5].isdigit(): + break; + iCpu = int(asWords[5]); + + # Add it and advance. + dRet[iCpu] = [sLine.rstrip() for sLine in asLines[2:-1]] + off = offEnd; + return dRet; + + def investigateInfoKvmLockSpinning(self, oCaseFile, sInfoText, dLogs): + """ Investigates kvm_lock_spinning deadlocks """ + # + # Extract the stacks. We need more than one CPU to create a deadlock. + # + dStacks = self.extractGuestCpuStack(sInfoText); + self.dprint('kvm_lock_spinning: found %s stacks' % (len(dStacks),)); + if len(dStacks) >= 2: + # + # Examin each of the stacks. Each must have kvm_lock_spinning in + # one of the first three entries. + # + cHits = 0; + for iCpu in dStacks: + asBacktrace = dStacks[iCpu]; + for iFrame in xrange(min(3, len(asBacktrace))): + if asBacktrace[iFrame].find('kvm_lock_spinning') >= 0: + cHits += 1; + break; + self.dprint('kvm_lock_spinning: %s/%s hits' % (cHits, len(dStacks),)); + if cHits == len(dStacks): + return (True, self.ktReason_VMM_kvm_lock_spinning); + + _ = dLogs; _ = oCaseFile; + return (False, None); + + def investigateInfoHalReturnToFirmware(self, oCaseFile, sInfoText, dLogs): + """ Investigates HalReturnToFirmware hangs """ + del oCaseFile + del sInfoText + del dLogs + # hope that's sufficient + return (True, self.ktReason_Unknown_HalReturnToFirmware); + + ## Things we search a main or VM log for to figure out why something went bust. + katSimpleMainAndVmLogReasons = [ + # ( Whether to stop on hit, reason tuple, needle text. ) + ( False, ktReason_Guru_Generic, 'GuruMeditation' ), + ( False, ktReason_Guru_Generic, 'Guru Meditation' ), + ( True, ktReason_Guru_VERR_IEM_INSTR_NOT_IMPLEMENTED, 'VERR_IEM_INSTR_NOT_IMPLEMENTED' ), + ( True, ktReason_Guru_VERR_IEM_ASPECT_NOT_IMPLEMENTED, 'VERR_IEM_ASPECT_NOT_IMPLEMENTED' ), + ( True, ktReason_Guru_VERR_TRPM_DONT_PANIC, 'VERR_TRPM_DONT_PANIC' ), + ( True, ktReason_Guru_VERR_PGM_PHYS_PAGE_RESERVED, 'VERR_PGM_PHYS_PAGE_RESERVED' ), + ( True, ktReason_Guru_VERR_VMX_INVALID_GUEST_STATE, 'VERR_VMX_INVALID_GUEST_STATE' ), + ( True, ktReason_Guru_VINF_EM_TRIPLE_FAULT, 'VINF_EM_TRIPLE_FAULT' ), + ( True, ktReason_Networking_Nonexistent_host_nic, + 'rc=E_FAIL text="Nonexistent host networking interface, name \'eth0\' (VERR_INTERNAL_ERROR)"' ), + ( True, ktReason_Host_Reboot_OSX_Watchdog_Timeout, ': "OSX Watchdog Timeout: ' ), + ( False, ktReason_XPCOM_NS_ERROR_CALL_FAILED, + 'Exception: 0x800706be (Call to remote object failed (NS_ERROR_CALL_FAILED))' ), + ( True, ktReason_Host_HostMemoryLow, 'HostMemoryLow' ), + ( True, ktReason_Host_HostMemoryLow, 'Failed to procure handy pages; rc=VERR_NO_MEMORY' ), + ( True, ktReason_Unknown_File_Not_Found, + 'Error: failed to start machine. Error message: File not found. (VERR_FILE_NOT_FOUND)' ), + ( True, ktReason_Unknown_File_Not_Found, # lump it in with file-not-found for now. + 'Error: failed to start machine. Error message: Not supported. (VERR_NOT_SUPPORTED)' ), + ( False, ktReason_Unknown_VM_Crash, 'txsDoConnectViaTcp: Machine state: Aborted' ), + ( True, ktReason_Host_Modprobe_Failed, 'Kernel driver not installed' ), + ( True, ktReason_OSInstall_Sata_no_BM, 'PCHS=14128/14134/8224' ), + ( True, ktReason_Host_DoubleFreeHeap, 'double free or corruption' ), + ( False, ktReason_Unknown_VM_Start_Error, 'VMSetError: ' ), + ( False, ktReason_Unknown_VM_Runtime_Error, 'Console: VM runtime error: fatal=true' ), + ]; + + ## Things we search a VBoxHardening.log file for to figure out why something went bust. + katSimpleVBoxHardeningLogReasons = [ + # ( Whether to stop on hit, reason tuple, needle text. ) + ( True, ktReason_Host_DriverNotLoaded, 'Error opening VBoxDrvStub: STATUS_OBJECT_NAME_NOT_FOUND' ), + ( True, ktReason_Host_NotSignedWithBuildCert, 'Not signed with the build certificate' ), + ]; + + ## Things we search a kernel.log file for to figure out why something went bust. + katSimpleKernelLogReasons = [ + # ( Whether to stop on hit, reason tuple, needle text. ) + ( True, ktReason_Panic_HugeMemory, 'mm/huge_memory.c:1988' ), + ( True, ktReason_Panic_IOAPICDoesntWork, 'IO-APIC + timer doesn''t work' ), + ( True, ktReason_Panic_TxUnitHang, 'Detected Tx Unit Hang' ), + ( True, ktReason_GuestBug_CompizVBoxQt, 'error 4 in libQt5CoreVBox' ), + ( True, ktReason_GuestBug_CompizVBoxQt, 'error 4 in libgtk-3' ), + ]; + + ## Things we search the _RIGHT_ _STRIPPED_ vgatext for. + katSimpleVgaTextReasons = [ + # ( Whether to stop on hit, reason tuple, needle text. ) + ( True, ktReason_Panic_MP_BIOS_IO_APIC, + "..MP-BIOS bug: 8254 timer not connected to IO-APIC\n\n" ), + ( True, ktReason_Panic_MP_BIOS_IO_APIC, + "..MP-BIOS bug: 8254 timer not connected to IO-APIC\n" + "...trying to set up timer (IRQ0) through the 8259A ... failed.\n" + "...trying to set up timer as Virtual Wire IRQ... failed.\n" + "...trying to set up timer as ExtINT IRQ... failed :(.\n" + "Kernel panic - not syncing: IO-APIC + timer doesn't work! Boot with apic=debug\n" + "and send a report. Then try booting with the 'noapic' option\n" + "\n" ), + ( True, ktReason_OSInstall_GRUB_hang, + "-----\nGRUB Loading stage2..\n\n\n\n" ), + ( True, ktReason_OSInstall_GRUB_hang, + "-----\nGRUB Loading stage2...\n\n\n\n" ), # the 3 dot hang appears to be less frequent + ( True, ktReason_OSInstall_GRUB_hang, + "-----\nGRUB Loading stage2....\n\n\n\n" ), # the 4 dot hang appears to be very infrequent + ( True, ktReason_OSInstall_GRUB_hang, + "-----\nGRUB Loading stage2.....\n\n\n\n" ), # the 5 dot hang appears to be more frequent again + ( True, ktReason_OSInstall_Udev_hang, + "\nStarting udev:\n\n\n\n" ), + ( True, ktReason_OSInstall_Udev_hang, + "\nStarting udev:\n------" ), + ( True, ktReason_Panic_BootManagerC000000F, + "Windows failed to start. A recent hardware or software change might be the" ), + ( True, ktReason_BootManager_Image_corrupt, + "BOOTMGR image is corrupt. The system cannot boot." ), + ]; + + ## Things we search for in the info.txt file. Require handlers for now. + katInfoTextHandlers = [ + # ( Trigger text, handler method ) + ( "kvm_lock_spinning", investigateInfoKvmLockSpinning ), + ( "HalReturnToFirmware", investigateInfoHalReturnToFirmware ), + ]; + + ## Mapping screenshot/failure SHA-256 hashes to failure reasons. + katSimpleScreenshotHashReasons = [ + # ( Whether to stop on hit, reason tuple, lowercased sha-256 of PIL.Image.tostring output ) + ( True, ktReason_BSOD_Recovery, '576f8e38d62b311cac7e3dc3436a0d0b9bd8cfd7fa9c43aafa95631520a45eac' ), + ( True, ktReason_BSOD_Automatic_Repair, 'c6a72076cc619937a7a39cfe9915b36d94cee0d4e3ce5ce061485792dcee2749' ), + ( True, ktReason_BSOD_Automatic_Repair, '26c4d8a724ff2c5e1051f3d5b650dbda7b5fdee0aa3e3c6059797f7484a515df' ), + ( True, ktReason_BSOD_0000007F, '57e1880619e13042a87100e7a38c8974b85ce3866501be621bea0cc696bb2c63' ), + ( True, ktReason_BSOD_000000D1, '134621281f00a3f8aeeb7660064bffbf6187ed56d5852142328d0bcb18ef0ede' ), + ( True, ktReason_BSOD_000000D1, '279f11258150c9d2fef041eca65501f3141da8df39256d8f6377e897e3b45a93' ), + ( True, ktReason_BSOD_C0000225, 'bd13a144be9dcdfb16bc863ff4c8f02a86e263c174f2cd5ffd27ca5f3aa31789' ), + ( True, ktReason_BSOD_C0000225, '8348b465e7ee9e59dd4e785880c57fd8677de05d11ac21e786bfde935307b42f' ), + ( True, ktReason_BSOD_C0000225, '1316e1fc818a73348412788e6910b8c016f237d8b4e15b20caf4a866f7a7840e' ), + ( True, ktReason_BSOD_C0000225, '54e0acbff365ce20a85abbe42bcd53647b8b9e80c68e45b2cd30e86bf177a0b5' ), + ( True, ktReason_BSOD_C0000225, '50fec50b5199923fa48b3f3e782687cc381e1c8a788ebda14e6a355fbe3bb1b3' ), + ]; + + def investigateVMResult(self, oCaseFile, oFailedResult, sResultLog): + """ + Investigates a failed VM run. + """ + + def investigateLogSet(): + """ + Investigates the current set of VM related logs. + """ + self.dprint('investigateLogSet: lengths: result log %u, VM log %u, kernel log %u, vga text %u, info text %u' + % ( len(sResultLog if sResultLog else ''), + len(sVMLog if sVMLog else ''), + len(sKrnlLog if sKrnlLog else ''), + len(sVgaText if sVgaText else ''), + len(sInfoText if sInfoText else ''), )); + + #self.dprint(u'main.log<<<\n%s\n<<<\n' % (sResultLog,)); + #self.dprint(u'vbox.log<<<\n%s\n<<<\n' % (sVMLog,)); + #self.dprint(u'krnl.log<<<\n%s\n<<<\n' % (sKrnlLog,)); + #self.dprint(u'vgatext.txt<<<\n%s\n<<<\n' % (sVgaText,)); + #self.dprint(u'info.txt<<<\n%s\n<<<\n' % (sInfoText,)); + + # TODO: more + + # + # Look for BSODs. Some stupid stupid inconsistencies in reason and log messages here, so don't try prettify this. + # + sDetails = self.findInAnyAndReturnRestOfLine([ sVMLog, sResultLog ], + 'GIM: HyperV: Guest indicates a fatal condition! P0='); + if sDetails is not None: + # P0=%#RX64 P1=%#RX64 P2=%#RX64 P3=%#RX64 P4=%#RX64 " + sKey = sDetails.split(' ', 1)[0]; + try: sKey = '0x%08X' % (int(sKey, 16),); + except: pass; + if sKey in self.asBsodReasons: + tReason = ( self.ksBsodCategory, sKey ); + elif sKey.lower() in self.asBsodReasons: # just in case. + tReason = ( self.ksBsodCategory, sKey.lower() ); + else: + self.dprint(u'BSOD "%s" not found in %s;' % (sKey, self.asBsodReasons)); + tReason = ( self.ksBsodCategory, self.ksBsodAddNew ); + return oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult, sComment = sDetails.strip()); + + # + # Look for linux panic. + # + if sKrnlLog is not None: + for fStopOnHit, tReason, sNeedle in self.katSimpleKernelLogReasons: + if sKrnlLog.find(sNeedle) > 0: + oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult); + if fStopOnHit: + return True; + fFoundSomething = True; + + # + # Loop thru the simple stuff. + # + fFoundSomething = False; + for fStopOnHit, tReason, sNeedle in self.katSimpleMainAndVmLogReasons: + if sResultLog.find(sNeedle) > 0 or (sVMLog is not None and sVMLog.find(sNeedle) > 0): + oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult); + if fStopOnHit: + return True; + fFoundSomething = True; + + # Continue with vga text. + if sVgaText: + for fStopOnHit, tReason, sNeedle in self.katSimpleVgaTextReasons: + if sVgaText.find(sNeedle) > 0: + oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult); + if fStopOnHit: + return True; + fFoundSomething = True; + _ = sInfoText; + + # Continue with screen hashes. + if sScreenHash is not None: + for fStopOnHit, tReason, sHash in self.katSimpleScreenshotHashReasons: + if sScreenHash == sHash: + oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult); + if fStopOnHit: + return True; + fFoundSomething = True; + + # Check VBoxHardening.log. + if sNtHardLog is not None: + for fStopOnHit, tReason, sNeedle in self.katSimpleVBoxHardeningLogReasons: + if sNtHardLog.find(sNeedle) > 0: + oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult); + if fStopOnHit: + return True; + fFoundSomething = True; + + # + # Complicated stuff. + # + dLogs = { + 'sVMLog': sVMLog, + 'sNtHardLog': sNtHardLog, + 'sScreenHash': sScreenHash, + 'sKrnlLog': sKrnlLog, + 'sVgaText': sVgaText, + 'sInfoText': sInfoText, + }; + + # info.txt. + if sInfoText: + for sNeedle, fnHandler in self.katInfoTextHandlers: + if sInfoText.find(sNeedle) > 0: + (fStop, tReason) = fnHandler(self, oCaseFile, sInfoText, dLogs); + if tReason is not None: + oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult); + if fStop: + return True; + fFoundSomething = True; + + # + # Check for repeated reboots... + # + if sVMLog is not None: + cResets = sVMLog.count('Changing the VM state from \'RUNNING\' to \'RESETTING\''); + if cResets > 10: + return oCaseFile.noteReasonForId(self.ktReason_Unknown_Reboot_Loop, oFailedResult.idTestResult, + sComment = 'Counted %s reboots' % (cResets,)); + + return fFoundSomething; + + # + # Check if we got any VM or/and kernel logs. Treat them as sets in + # case we run multiple VMs here (this is of course ASSUMING they + # appear in the order that terminateVmBySession uploads them). + # + cTimes = 0; + sVMLog = None; + sNtHardLog = None; + sScreenHash = None; + sKrnlLog = None; + sVgaText = None; + sInfoText = None; + for oFile in oFailedResult.aoFiles: + if oFile.sKind == TestResultFileData.ksKind_LogReleaseVm: + if 'VBoxHardening.log' not in oFile.sFile: + if sVMLog is not None: + if investigateLogSet() is True: + return True; + cTimes += 1; + sInfoText = None; + sVgaText = None; + sKrnlLog = None; + sScreenHash = None; + sNtHardLog = None; + sVMLog = oCaseFile.getLogFile(oFile); + else: + sNtHardLog = oCaseFile.getLogFile(oFile); + elif oFile.sKind == TestResultFileData.ksKind_LogGuestKernel: + sKrnlLog = oCaseFile.getLogFile(oFile); + elif oFile.sKind == TestResultFileData.ksKind_InfoVgaText: + sVgaText = '\n'.join([sLine.rstrip() for sLine in oCaseFile.getLogFile(oFile).split('\n')]); + elif oFile.sKind == TestResultFileData.ksKind_InfoCollection: + sInfoText = oCaseFile.getLogFile(oFile); + elif oFile.sKind == TestResultFileData.ksKind_ScreenshotFailure: + sScreenHash = oCaseFile.getScreenshotSha256(oFile); + if sScreenHash is not None: + sScreenHash = sScreenHash.lower(); + self.vprint(u'%s %s' % ( sScreenHash, oFile.sFile,)); + + if ( sVMLog is not None \ + or sNtHardLog is not None \ + or cTimes == 0) \ + and investigateLogSet() is True: + return True; + + return None; + + + def isResultFromVMRun(self, oFailedResult, sResultLog): + """ + Checks if this result and corresponding log snippet looks like a VM run. + """ + + # Look for startVmEx/ startVmAndConnectToTxsViaTcp and similar output in the log. + if sResultLog.find(' startVm') > 0: + return True; + + # Any other indicators? No? + _ = oFailedResult; + return False; + + def investigateVBoxVMTest(self, oCaseFile, fSingleVM): + """ + Checks out a VBox VM test. + + This is generic investigation of a test running one or more VMs, like + for example a smoke test or a guest installation test. + + The fSingleVM parameter is a hint, which probably won't come in useful. + """ + _ = fSingleVM; + + # + # Get a list of test result failures we should be looking into and the main log. + # + aoFailedResults = oCaseFile.oTree.getListOfFailures(); + sMainLog = oCaseFile.getMainLog(); + + # + # There are a set of errors ending up on the top level result record. + # Should deal with these first. + # + if len(aoFailedResults) == 1 and aoFailedResults[0] == oCaseFile.oTree: + # Check if we've just got that XPCOM client smoke test shutdown issue. This will currently always + # be reported on the top result because vboxinstall.py doesn't add an error for it. It is easy to + # ignore other failures in the test if we're not a little bit careful here. + if sMainLog.find('vboxinstaller: Exit code: -11 (') > 0: + oCaseFile.noteReason(self.ktReason_XPCOM_Exit_Minus_11); + return self.caseClosed(oCaseFile); + + # Hang after starting VBoxSVC (e.g. idTestSet=136307258) + if self.isThisFollowedByTheseLines(sMainLog, 'oVBoxMgr=<vboxapi.VirtualBoxManager object at', + (' Timeout: ', ' Attempting to abort child...',) ): + if sMainLog.find('*** glibc detected *** /') > 0: + oCaseFile.noteReason(self.ktReason_XPCOM_VBoxSVC_Hang_Plus_Heap_Corruption); + else: + oCaseFile.noteReason(self.ktReason_XPCOM_VBoxSVC_Hang); + return self.caseClosed(oCaseFile); + + # Look for heap corruption without visible hang. + if sMainLog.find('*** glibc detected *** /') > 0 \ + or sMainLog.find("-1073740940") > 0: # STATUS_HEAP_CORRUPTION / 0xc0000374 + oCaseFile.noteReason(self.ktReason_Unknown_Heap_Corruption); + return self.caseClosed(oCaseFile); + + # Out of memory w/ timeout. + if sMainLog.find('sErrId=HostMemoryLow') > 0: + oCaseFile.noteReason(self.ktReason_Host_HostMemoryLow); + return self.caseClosed(oCaseFile); + + # Stale files like vts_rm.exe (windows). + offEnd = sMainLog.rfind('*** The test driver exits successfully. ***'); + if offEnd > 0 and sMainLog.find('[Error 145] The directory is not empty: ', offEnd) > 0: + oCaseFile.noteReason(self.ktReason_Ignore_Stale_Files); + return self.caseClosed(oCaseFile); + + # + # XPCOM screwup + # + if sMainLog.find('AttributeError: \'NoneType\' object has no attribute \'addObserver\'') > 0: + oCaseFile.noteReason(self.ktReason_Buggy_Build_Broken_Build); + return self.caseClosed(oCaseFile); + + # + # Go thru each failed result. + # + for oFailedResult in aoFailedResults: + self.dprint(u'Looking at test result #%u - %s' % (oFailedResult.idTestResult, oFailedResult.getFullName(),)); + sResultLog = TestSetData.extractLogSectionElapsed(sMainLog, oFailedResult.tsCreated, oFailedResult.tsElapsed); + if oFailedResult.sName == 'Installing VirtualBox': + self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = True) + + elif oFailedResult.sName == 'Uninstalling VirtualBox': + self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = False) + + elif self.isResultFromVMRun(oFailedResult, sResultLog): + self.investigateVMResult(oCaseFile, oFailedResult, sResultLog); + + elif sResultLog.find('most likely not unique') > 0: + oCaseFile.noteReasonForId(self.ktReason_Host_NetworkMisconfiguration, oFailedResult.idTestResult) + elif sResultLog.find('Exception: 0x800706be (Call to remote object failed (NS_ERROR_CALL_FAILED))') > 0: + oCaseFile.noteReasonForId(self.ktReason_XPCOM_NS_ERROR_CALL_FAILED, oFailedResult.idTestResult); + + elif sResultLog.find('The machine is not mutable (state is ') > 0: + self.vprint('Ignoring "machine not mutable" error as it is probably due to an earlier problem'); + oCaseFile.noteReasonForId(self.ktHarmless, oFailedResult.idTestResult); + + elif sResultLog.find('** error: no action was specified') > 0 \ + or sResultLog.find('(len(self._asXml, asText))') > 0: + oCaseFile.noteReasonForId(self.ktReason_Ignore_Buggy_Test_Driver, oFailedResult.idTestResult); + + else: + self.vprint(u'TODO: Cannot place idTestResult=%u - %s' % (oFailedResult.idTestResult, oFailedResult.sName,)); + self.dprint(u'%s + %s <<\n%s\n<<' % (oFailedResult.tsCreated, oFailedResult.tsElapsed, sResultLog,)); + + # + # Report home and close the case if we got them all, otherwise log it. + # + if len(oCaseFile.dReasonForResultId) >= len(aoFailedResults): + return self.caseClosed(oCaseFile); + + if oCaseFile.dReasonForResultId: + self.vprint(u'TODO: Got %u out of %u - close, but no cigar. :-/' + % (len(oCaseFile.dReasonForResultId), len(aoFailedResults))); + else: + self.vprint(u'XXX: Could not figure out anything at all! :-('); + return False; + + + def reasoningFailures(self): + """ + Guess the reason for failures. + """ + # + # Get a list of failed test sets without any assigned failure reason. + # + cGot = 0; + aoTestSets = self.oTestSetLogic.fetchFailedSetsWithoutReason(cHoursBack = self.oConfig.cHoursBack, tsNow = self.tsNow); + for oTestSet in aoTestSets: + self.dprint(u''); + self.dprint(u'reasoningFailures: Checking out test set #%u, status %s' % ( oTestSet.idTestSet, oTestSet.enmStatus,)) + + # + # Open a case file and assign it to the right investigator. + # + (oTree, _ ) = self.oTestResultLogic.fetchResultTree(oTestSet.idTestSet); + oBuild = BuildDataEx().initFromDbWithId( self.oDb, oTestSet.idBuild, oTestSet.tsCreated); + oTestBox = TestBoxData().initFromDbWithGenId( self.oDb, oTestSet.idGenTestBox); + oTestGroup = TestGroupData().initFromDbWithId( self.oDb, oTestSet.idTestGroup, oTestSet.tsCreated); + oTestCase = TestCaseDataEx().initFromDbWithGenId( self.oDb, oTestSet.idGenTestCase, oTestSet.tsConfig); + + oCaseFile = VirtualTestSheriffCaseFile(self, oTestSet, oTree, oBuild, oTestBox, oTestGroup, oTestCase); + + if oTestSet.enmStatus == TestSetData.ksTestStatus_BadTestBox: + self.dprint(u'investigateBadTestBox is taking over %s.' % (oCaseFile.sLongName,)); + fRc = self.investigateBadTestBox(oCaseFile); + + elif oCaseFile.isVBoxUnitTest(): + self.dprint(u'investigateVBoxUnitTest is taking over %s.' % (oCaseFile.sLongName,)); + fRc = self.investigateVBoxUnitTest(oCaseFile); + + elif oCaseFile.isVBoxInstallTest(): + self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,)); + fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True); + + elif oCaseFile.isVBoxUSBTest(): + self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,)); + fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True); + + elif oCaseFile.isVBoxStorageTest(): + self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,)); + fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True); + + elif oCaseFile.isVBoxGAsTest(): + self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,)); + fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True); + + elif oCaseFile.isVBoxAPITest(): + self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,)); + fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True); + + elif oCaseFile.isVBoxBenchmarkTest(): + self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,)); + fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = False); + + elif oCaseFile.isVBoxSmokeTest(): + self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,)); + fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = False); + + else: + self.vprint(u'reasoningFailures: Unable to classify test set: %s' % (oCaseFile.sLongName,)); + fRc = False; + cGot += fRc is True; + + self.vprint(u'reasoningFailures: Got %u out of %u' % (cGot, len(aoTestSets), )); + return 0; + + + def main(self): + """ + The 'main' function. + Return exit code (0, 1, etc). + """ + # Database stuff. + self.oDb = TMDatabaseConnection() + self.oTestResultLogic = TestResultLogic(self.oDb); + self.oTestSetLogic = TestSetLogic(self.oDb); + self.oFailureReasonLogic = FailureReasonLogic(self.oDb); + self.oTestResultFailureLogic = TestResultFailureLogic(self.oDb); + self.asBsodReasons = self.oFailureReasonLogic.fetchForSheriffByNamedCategory(self.ksBsodCategory); + self.asUnitTestReasons = self.oFailureReasonLogic.fetchForSheriffByNamedCategory(self.ksUnitTestCategory); + + # Get a fix on our 'now' before we do anything.. + self.oDb.execute('SELECT CURRENT_TIMESTAMP - interval \'%s hours\'', (self.oConfig.cStartHoursAgo,)); + self.tsNow = self.oDb.fetchOne(); + + # If we're suppost to commit anything we need to get our user ID. + rcExit = 0; + if self.oConfig.fRealRun: + self.oLogin = UserAccountLogic(self.oDb).tryFetchAccountByLoginName(VirtualTestSheriff.ksLoginName); + if self.oLogin is None: + rcExit = self.eprint('Cannot find my user account "%s"!' % (VirtualTestSheriff.ksLoginName,)); + else: + self.uidSelf = self.oLogin.uid; + + # + # Do the stuff. + # + if rcExit == 0: + rcExit = self.selfCheck(); + if rcExit == 0: + rcExit = self.badTestBoxManagement(); + rcExit2 = self.reasoningFailures(); + if rcExit == 0: + rcExit = rcExit2; + # Redo the bad testbox management after failure reasons have been assigned (got timing issues). + if rcExit == 0: + rcExit = self.badTestBoxManagement(); + + # Cleanup. + self.oFailureReasonLogic = None; + self.oTestResultFailureLogic = None; + self.oTestSetLogic = None; + self.oTestResultLogic = None; + self.oDb.close(); + self.oDb = None; + if self.oLogFile is not None: + self.oLogFile.close(); + self.oLogFile = None; + return rcExit; + +if __name__ == '__main__': + sys.exit(VirtualTestSheriff().main()); + |