From f215e02bf85f68d3a6106c2a1f4f7f063f819064 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Thu, 11 Apr 2024 10:17:27 +0200 Subject: Adding upstream version 7.0.14-dfsg. Signed-off-by: Daniel Baumann --- .../ValidationKit/testmanager/batch/Makefile.kmk | 46 + .../ValidationKit/testmanager/batch/add_build.py | 137 ++ .../testmanager/batch/check_for_deleted_builds.py | 133 ++ .../testmanager/batch/close_orphaned_testsets.py | 105 ++ .../ValidationKit/testmanager/batch/del_build.py | 95 + .../testmanager/batch/filearchiver.py | 282 +++ src/VBox/ValidationKit/testmanager/batch/quota.py | 319 ++++ .../testmanager/batch/regen_sched_queues.py | 132 ++ .../ValidationKit/testmanager/batch/vcs_import.py | 205 +++ .../testmanager/batch/virtual_test_sheriff.py | 1832 ++++++++++++++++++++ 10 files changed, 3286 insertions(+) create mode 100644 src/VBox/ValidationKit/testmanager/batch/Makefile.kmk create mode 100755 src/VBox/ValidationKit/testmanager/batch/add_build.py create mode 100755 src/VBox/ValidationKit/testmanager/batch/check_for_deleted_builds.py create mode 100755 src/VBox/ValidationKit/testmanager/batch/close_orphaned_testsets.py create mode 100755 src/VBox/ValidationKit/testmanager/batch/del_build.py create mode 100755 src/VBox/ValidationKit/testmanager/batch/filearchiver.py create mode 100755 src/VBox/ValidationKit/testmanager/batch/quota.py create mode 100755 src/VBox/ValidationKit/testmanager/batch/regen_sched_queues.py create mode 100755 src/VBox/ValidationKit/testmanager/batch/vcs_import.py create mode 100755 src/VBox/ValidationKit/testmanager/batch/virtual_test_sheriff.py (limited to 'src/VBox/ValidationKit/testmanager/batch') diff --git a/src/VBox/ValidationKit/testmanager/batch/Makefile.kmk b/src/VBox/ValidationKit/testmanager/batch/Makefile.kmk new file mode 100644 index 00000000..74d882cc --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/batch/Makefile.kmk @@ -0,0 +1,46 @@ +# $Id: Makefile.kmk $ +## @file +# VirtualBox Validation Kit. +# + +# +# Copyright (C) 2006-2023 Oracle and/or its affiliates. +# +# This file is part of VirtualBox base platform packages, as +# available from https://www.virtualbox.org. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation, in version 3 of the +# License. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . +# +# The contents of this file may alternatively be used under the terms +# of the Common Development and Distribution License Version 1.0 +# (CDDL), a copy of it is provided in the "COPYING.CDDL" file included +# in the VirtualBox distribution, in which case the provisions of the +# CDDL are applicable instead of those of the GPL. +# +# You may elect to license modified versions of this file under the +# terms and conditions of either the GPL or the CDDL or both. +# +# SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +# + +SUB_DEPTH = ../../../../.. +include $(KBUILD_PATH)/subheader.kmk + + +VBOX_VALIDATIONKIT_PYTHON_SOURCES += $(wildcard $(PATH_SUB_CURRENT)/*.py) + +$(evalcall def_vbox_validationkit_process_python_sources) +$(evalcall def_vbox_validationkit_process_js_sources) +include $(FILE_KBUILD_SUB_FOOTER) + diff --git a/src/VBox/ValidationKit/testmanager/batch/add_build.py b/src/VBox/ValidationKit/testmanager/batch/add_build.py new file mode 100755 index 00000000..a82da5eb --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/batch/add_build.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# $Id: add_build.py $ +# pylint: disable=line-too-long + +""" +Interface used by the tinderbox server side software to add a fresh build. +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see . + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + +# Standard python imports +import sys; +import os; +from optparse import OptionParser; # pylint: disable=deprecated-module + +# Add Test Manager's modules path +g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))); +sys.path.append(g_ksTestManagerDir); + +# Test Manager imports +from testmanager.core.db import TMDatabaseConnection; +from testmanager.core.build import BuildDataEx, BuildLogic, BuildCategoryData; + +class Build(object): # pylint: disable=too-few-public-methods + """ + Add build info into Test Manager database. + """ + + def __init__(self): + """ + Parse command line. + """ + + oParser = OptionParser(); + oParser.add_option('-q', '--quiet', dest = 'fQuiet', action = 'store_true', + help = 'Quiet execution'); + oParser.add_option('-b', '--branch', dest = 'sBranch', metavar = '', + help = 'branch name (default: trunk)', default = 'trunk'); + oParser.add_option('-p', '--product', dest = 'sProductName', metavar = '', + help = 'The product name.'); + oParser.add_option('-r', '--revision', dest = 'iRevision', metavar = '', + help = 'revision number'); + oParser.add_option('-R', '--repository', dest = 'sRepository', metavar = '', + help = 'Version control repository name.'); + oParser.add_option('-t', '--type', dest = 'sBuildType', metavar = '', + help = 'build type (debug, release etc.)'); + oParser.add_option('-v', '--version', dest = 'sProductVersion', metavar = '', + help = 'The product version number (suitable for RTStrVersionCompare)'); + oParser.add_option('-o', '--os-arch', dest = 'asTargetOsArches', metavar = '', action = 'append', + help = 'Target OS and architecture. This option can be repeated.'); + oParser.add_option('-l', '--log', dest = 'sBuildLogPath', metavar = '', + help = 'URL to the build logs (optional).'); + oParser.add_option('-f', '--file', dest = 'asFiles', metavar = '', action = 'append', + help = 'URLs or build share relative path to a build output file. This option can be repeated.'); + + (self.oConfig, _) = oParser.parse_args(); + + # Check command line + asMissing = []; + if self.oConfig.sBranch is None: asMissing.append('--branch'); + if self.oConfig.iRevision is None: asMissing.append('--revision'); + if self.oConfig.sProductVersion is None: asMissing.append('--version'); + if self.oConfig.sProductName is None: asMissing.append('--product'); + if self.oConfig.sBuildType is None: asMissing.append('--type'); + if self.oConfig.asTargetOsArches is None: asMissing.append('--os-arch'); + if self.oConfig.asFiles is None: asMissing.append('--file'); + if asMissing: + sys.stderr.write('syntax error: Missing: %s\n' % (asMissing,)); + sys.exit(1); + # Temporary default. + if self.oConfig.sRepository is None: + self.oConfig.sRepository = 'vbox'; + + def add(self): + """ + Add build data record into database. + """ + oDb = TMDatabaseConnection() + + # Assemble the build data. + oBuildData = BuildDataEx() + oBuildData.idBuildCategory = None; + oBuildData.iRevision = self.oConfig.iRevision + oBuildData.sVersion = self.oConfig.sProductVersion + oBuildData.sLogUrl = self.oConfig.sBuildLogPath + oBuildData.sBinaries = ','.join(self.oConfig.asFiles); + oBuildData.oCat = BuildCategoryData().initFromValues(sProduct = self.oConfig.sProductName, + sRepository = self.oConfig.sRepository, + sBranch = self.oConfig.sBranch, + sType = self.oConfig.sBuildType, + asOsArches = self.oConfig.asTargetOsArches); + + # Add record to database + try: + BuildLogic(oDb).addEntry(oBuildData, fCommit = True); + except: + if self.oConfig.fQuiet: + sys.exit(1); + raise; + oDb.close(); + return 0; + +if __name__ == '__main__': + sys.exit(Build().add()); + diff --git a/src/VBox/ValidationKit/testmanager/batch/check_for_deleted_builds.py b/src/VBox/ValidationKit/testmanager/batch/check_for_deleted_builds.py new file mode 100755 index 00000000..1ac5ab95 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/batch/check_for_deleted_builds.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# $Id: check_for_deleted_builds.py $ +# pylint: disable=line-too-long + +""" +Admin job for checking detecting deleted builds. + +This is necessary when the tinderbox <-> test manager interface was +busted and the build info in is out of sync. The result is generally +a lot of skipped tests because of missing builds, typically during +bisecting problems. +""" + +from __future__ import print_function; + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see . + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + +# Standard python imports +import sys; +import os; +from optparse import OptionParser; # pylint: disable=deprecated-module + +# Add Test Manager's modules path +g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))); +sys.path.append(g_ksTestManagerDir); + +# Test Manager imports +from testmanager.core.db import TMDatabaseConnection; +from testmanager.core.build import BuildLogic; + + + +class BuildChecker(object): # pylint: disable=too-few-public-methods + """ + Add build info into Test Manager database. + """ + + def __init__(self): + """ + Parse command line. + """ + + oParser = OptionParser(); + oParser.add_option('-q', '--quiet', dest = 'fQuiet', action = 'store_true', default = False, + help = 'Quiet execution'); + oParser.add_option('--dry-run', dest = 'fRealRun', action = 'store_false', default = False, + help = 'Dry run'); + oParser.add_option('--real-run', dest = 'fRealRun', action = 'store_true', default = False, + help = 'Real run'); + + (self.oConfig, _) = oParser.parse_args(); + if not self.oConfig.fQuiet: + if not self.oConfig.fRealRun: + print('Dry run.'); + else: + print('Real run! Will commit findings!'); + + + def checkBuilds(self): + """ + Add build data record into database. + """ + oDb = TMDatabaseConnection(); + oBuildLogic = BuildLogic(oDb); + + tsNow = oDb.getCurrentTimestamp(); + cMaxRows = 1024; + iStart = 0; + while True: + aoBuilds = oBuildLogic.fetchForListing(iStart, cMaxRows, tsNow); + if not self.oConfig.fQuiet and aoBuilds: + print('Processing builds #%s thru #%s' % (aoBuilds[0].idBuild, aoBuilds[-1].idBuild)); + + for oBuild in aoBuilds: + if oBuild.fBinariesDeleted is False: + rc = oBuild.areFilesStillThere(); + if rc is False: + if not self.oConfig.fQuiet: + print('missing files for build #%s / r%s / %s / %s / %s / %s / %s' + % (oBuild.idBuild, oBuild.iRevision, oBuild.sVersion, oBuild.oCat.sType, + oBuild.oCat.sBranch, oBuild.oCat.sProduct, oBuild.oCat.asOsArches,)); + print(' %s' % (oBuild.sBinaries,)); + if self.oConfig.fRealRun is True: + oBuild.fBinariesDeleted = True; + oBuildLogic.editEntry(oBuild, fCommit = True); + elif rc is True and not self.oConfig.fQuiet: + print('build #%s still have its files' % (oBuild.idBuild,)); + elif rc is None and not self.oConfig.fQuiet: + print('Unable to determine state of build #%s' % (oBuild.idBuild,)); + + # advance + if len(aoBuilds) < cMaxRows: + break; + iStart += len(aoBuilds); + + oDb.close(); + return 0; + +if __name__ == '__main__': + sys.exit(BuildChecker().checkBuilds()); + diff --git a/src/VBox/ValidationKit/testmanager/batch/close_orphaned_testsets.py b/src/VBox/ValidationKit/testmanager/batch/close_orphaned_testsets.py new file mode 100755 index 00000000..932122b5 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/batch/close_orphaned_testsets.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# $Id: close_orphaned_testsets.py $ +# pylint: disable=line-too-long + +""" +Maintenance tool for closing orphaned testsets. +""" + +from __future__ import print_function; + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see . + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + +# Standard python imports +import sys +import os +from optparse import OptionParser; # pylint: disable=deprecated-module + +# Add Test Manager's modules path +g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.append(g_ksTestManagerDir) + +# Test Manager imports +from testmanager.core.db import TMDatabaseConnection +from testmanager.core.testset import TestSetLogic; + + +class CloseOrphanedTestSets(object): + """ + Finds and closes orphaned testsets. + """ + + def __init__(self): + """ + Parse command line + """ + oParser = OptionParser(); + oParser.add_option('-d', '--just-do-it', dest='fJustDoIt', action='store_true', + help='Do the database changes.'); + + + (self.oConfig, _) = oParser.parse_args(); + + + def main(self): + """ Main method. """ + oDb = TMDatabaseConnection(); + + # Get a list of orphans. + oLogic = TestSetLogic(oDb); + aoOrphans = oLogic.fetchOrphaned(); + if aoOrphans: + # Complete them. + if self.oConfig.fJustDoIt: + print('Completing %u test sets as abandoned:' % (len(aoOrphans),)); + for oTestSet in aoOrphans: + print('#%-7u: idTestBox=%-3u tsCreated=%s tsDone=%s' + % (oTestSet.idTestSet, oTestSet.idTestBox, oTestSet.tsCreated, oTestSet.tsDone)); + oLogic.completeAsAbandoned(oTestSet.idTestSet); + print('Committing...'); + oDb.commit(); + else: + for oTestSet in aoOrphans: + print('#%-7u: idTestBox=%-3u tsCreated=%s tsDone=%s' + % (oTestSet.idTestSet, oTestSet.idTestBox, oTestSet.tsCreated, oTestSet.tsDone)); + print('Not completing any testsets without seeing the --just-do-it option.'); + else: + print('No orphaned test sets.\n'); + return 0; + + +if __name__ == '__main__': + sys.exit(CloseOrphanedTestSets().main()) + diff --git a/src/VBox/ValidationKit/testmanager/batch/del_build.py b/src/VBox/ValidationKit/testmanager/batch/del_build.py new file mode 100755 index 00000000..76e43344 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/batch/del_build.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# $Id: del_build.py $ +# pylint: disable=line-too-long + +""" +Interface used by the tinderbox server side software to mark build binaries +deleted. +""" + +from __future__ import print_function; + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see . + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + +# Standard python imports +import sys +import os +from optparse import OptionParser; # pylint: disable=deprecated-module + +# Add Test Manager's modules path +g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.append(g_ksTestManagerDir) + +# Test Manager imports +from testmanager.core.db import TMDatabaseConnection +from testmanager.core.build import BuildLogic + + +def markBuildsDeleted(): + """ + Marks the builds using the specified binaries as deleted. + """ + + oParser = OptionParser() + oParser.add_option('-q', '--quiet', dest='fQuiet', action='store_true', + help='Quiet execution'); + + (oConfig, asArgs) = oParser.parse_args() + if not asArgs: + if not oConfig.fQuiet: + sys.stderr.write('syntax error: No builds binaries specified\n'); + return 1; + + + oDb = TMDatabaseConnection() + oLogic = BuildLogic(oDb) + + for sBuildBin in asArgs: + try: + cBuilds = oLogic.markDeletedByBinaries(sBuildBin, fCommit = True) + except: + if oConfig.fQuiet: + sys.exit(1); + raise; + else: + if not oConfig.fQuiet: + print("del_build.py: Marked %u builds associated with '%s' as deleted." % (cBuilds, sBuildBin,)); + + oDb.close() + return 0; + +if __name__ == '__main__': + sys.exit(markBuildsDeleted()) + diff --git a/src/VBox/ValidationKit/testmanager/batch/filearchiver.py b/src/VBox/ValidationKit/testmanager/batch/filearchiver.py new file mode 100755 index 00000000..10a772ae --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/batch/filearchiver.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# $Id: filearchiver.py $ +# pylint: disable=line-too-long + +""" +A cronjob that compresses logs and other files, moving them to the +g_ksZipFileAreaRootDir storage area. +""" + +from __future__ import print_function; + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see . + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + +# Standard python imports +import sys +import os +from optparse import OptionParser; # pylint: disable=deprecated-module +import time; +import zipfile; + +# Add Test Manager's modules path +g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.append(g_ksTestManagerDir) + +# Test Manager imports +from common import utils; +from testmanager import config; +from testmanager.core.db import TMDatabaseConnection; +from testmanager.core.testset import TestSetData, TestSetLogic; + + + +class FileArchiverBatchJob(object): # pylint: disable=too-few-public-methods + """ + Log+files comp + """ + + def __init__(self, oOptions): + """ + Parse command line + """ + self.fVerbose = oOptions.fVerbose; + self.sSrcDir = config.g_ksFileAreaRootDir; + self.sDstDir = config.g_ksZipFileAreaRootDir; + #self.oTestSetLogic = TestSetLogic(TMDatabaseConnection(self.dprint if self.fVerbose else None)); + self.oTestSetLogic = TestSetLogic(TMDatabaseConnection(None)); + self.fDryRun = oOptions.fDryRun; + + def dprint(self, sText): + """ Verbose output. """ + if self.fVerbose: + print(sText); + return True; + + def warning(self, sText): + """Prints a warning.""" + print(sText); + return True; + + def _processTestSet(self, idTestSet, asFiles, sCurDir): + """ + Worker for processDir. + Same return codes as processDir. + """ + + sBaseFilename = os.path.join(sCurDir, 'TestSet-%d' % (idTestSet,)); + if sBaseFilename[0:2] == ('.' + os.path.sep): + sBaseFilename = sBaseFilename[2:]; + sSrcFileBase = os.path.join(self.sSrcDir, sBaseFilename + '-'); + + # + # Skip the file if the test set is still running. + # But delete them if the testset is not found. + # + oTestSet = self.oTestSetLogic.tryFetch(idTestSet); + if oTestSet is not None and sBaseFilename != oTestSet.sBaseFilename: + self.warning('TestSet %d: Deleting because sBaseFilename differs: "%s" (disk) vs "%s" (db)' \ + % (idTestSet, sBaseFilename, oTestSet.sBaseFilename,)); + oTestSet = None; + + if oTestSet is not None: + if oTestSet.enmStatus == TestSetData.ksTestStatus_Running: + self.dprint('Skipping test set #%d, still running' % (idTestSet,)); + return True; + + # + # If we have a zip file already, don't try recreate it as we might + # have had trouble removing the source files. + # + sDstDirPath = os.path.join(self.sDstDir, sCurDir); + sZipFileNm = os.path.join(sDstDirPath, 'TestSet-%d.zip' % (idTestSet,)); + if not os.path.exists(sZipFileNm): + # + # Create zip file with all testset files as members. + # + self.dprint('TestSet %d: Creating %s...' % (idTestSet, sZipFileNm,)); + if not self.fDryRun: + + if not os.path.exists(sDstDirPath): + os.makedirs(sDstDirPath, 0o755); + + utils.noxcptDeleteFile(sZipFileNm + '.tmp'); + with zipfile.ZipFile(sZipFileNm + '.tmp', 'w', zipfile.ZIP_DEFLATED, allowZip64 = True) as oZipFile: + for sFile in asFiles: + sSuff = os.path.splitext(sFile)[1]; + if sSuff in [ '.png', '.webm', '.gz', '.bz2', '.zip', '.mov', '.avi', '.mpg', '.gif', '.jpg' ]: + ## @todo Consider storing these files outside the zip if they are a little largish. + self.dprint('TestSet %d: Storing %s...' % (idTestSet, sFile)); + oZipFile.write(sSrcFileBase + sFile, sFile, zipfile.ZIP_STORED); + else: + self.dprint('TestSet %d: Deflating %s...' % (idTestSet, sFile)); + oZipFile.write(sSrcFileBase + sFile, sFile, zipfile.ZIP_DEFLATED); + + # + # .zip.tmp -> .zip. + # + utils.noxcptDeleteFile(sZipFileNm); + os.rename(sZipFileNm + '.tmp', sZipFileNm); + + #else: Dry run. + else: + self.dprint('TestSet %d: zip file exists already (%s)' % (idTestSet, sZipFileNm,)); + + # + # Delete the files. + # + fRc = True; + if self.fVerbose: + self.dprint('TestSet %d: deleting file: %s' % (idTestSet, asFiles)); + if not self.fDryRun: + for sFile in asFiles: + if utils.noxcptDeleteFile(sSrcFileBase + sFile) is False: + self.warning('TestSet %d: Failed to delete "%s" (%s)' % (idTestSet, sFile, sSrcFileBase + sFile,)); + fRc = False; + + return fRc; + + + def processDir(self, sCurDir): + """ + Process the given directory (relative to sSrcDir and sDstDir). + Returns success indicator. + """ + if self.fVerbose: + self.dprint('processDir: %s' % (sCurDir,)); + + # + # Sift thought the directory content, collecting subdirectories and + # sort relevant files by test set. + # Generally there will either be subdirs or there will be files. + # + asSubDirs = []; + dTestSets = {}; + sCurPath = os.path.abspath(os.path.join(self.sSrcDir, sCurDir)); + for sFile in os.listdir(sCurPath): + if os.path.isdir(os.path.join(sCurPath, sFile)): + if sFile not in [ '.', '..' ]: + asSubDirs.append(sFile); + elif sFile.startswith('TestSet-'): + # Parse the file name. ASSUMES 'TestSet-%d-filename' format. + iSlash1 = sFile.find('-'); + iSlash2 = sFile.find('-', iSlash1 + 1); + if iSlash2 <= iSlash1: + self.warning('Bad filename (1): "%s"' % (sFile,)); + continue; + + try: idTestSet = int(sFile[(iSlash1 + 1):iSlash2]); + except: + self.warning('Bad filename (2): "%s"' % (sFile,)); + if self.fVerbose: + self.dprint('\n'.join(utils.getXcptInfo(4))); + continue; + + if idTestSet <= 0: + self.warning('Bad filename (3): "%s"' % (sFile,)); + continue; + + if iSlash2 + 2 >= len(sFile): + self.warning('Bad filename (4): "%s"' % (sFile,)); + continue; + sName = sFile[(iSlash2 + 1):]; + + # Add it. + if idTestSet not in dTestSets: + dTestSets[idTestSet] = []; + asTestSet = dTestSets[idTestSet]; + asTestSet.append(sName); + + # + # Test sets. + # + fRc = True; + for idTestSet, oTestSet in dTestSets.items(): + try: + if self._processTestSet(idTestSet, oTestSet, sCurDir) is not True: + fRc = False; + except: + self.warning('TestSet %d: Exception in _processTestSet:\n%s' % (idTestSet, '\n'.join(utils.getXcptInfo()),)); + fRc = False; + + # + # Sub dirs. + # + for sSubDir in asSubDirs: + if self.processDir(os.path.join(sCurDir, sSubDir)) is not True: + fRc = False; + + # + # Try Remove the directory iff it's not '.' and it's been unmodified + # for the last 24h (race protection). + # + if sCurDir != '.': + try: + fpModTime = float(os.path.getmtime(sCurPath)); + if fpModTime + (24*3600) <= time.time(): + if utils.noxcptRmDir(sCurPath) is True: + self.dprint('Removed "%s".' % (sCurPath,)); + except: + pass; + + return fRc; + + @staticmethod + def main(): + """ C-style main(). """ + # + # Parse options. + # + oParser = OptionParser(); + oParser.add_option('-v', '--verbose', dest = 'fVerbose', action = 'store_true', default = False, + help = 'Verbose output.'); + oParser.add_option('-q', '--quiet', dest = 'fVerbose', action = 'store_false', default = False, + help = 'Quiet operation.'); + oParser.add_option('-d', '--dry-run', dest = 'fDryRun', action = 'store_true', default = False, + help = 'Dry run, do not make any changes.'); + (oOptions, asArgs) = oParser.parse_args() + if asArgs != []: + oParser.print_help(); + return 1; + + # + # Do the work. + # + oBatchJob = FileArchiverBatchJob(oOptions); + fRc = oBatchJob.processDir('.'); + return 0 if fRc is True else 1; + +if __name__ == '__main__': + sys.exit(FileArchiverBatchJob.main()); + diff --git a/src/VBox/ValidationKit/testmanager/batch/quota.py b/src/VBox/ValidationKit/testmanager/batch/quota.py new file mode 100755 index 00000000..e2854881 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/batch/quota.py @@ -0,0 +1,319 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# $Id: quota.py $ +# pylint: disable=line-too-long + +""" +A cronjob that applies quotas to large files in testsets. +""" + +from __future__ import print_function; + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see . + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + +# Standard python imports +import sys +import os +from optparse import OptionParser; # pylint: disable=deprecated-module +import shutil +import tempfile; +import zipfile; + +# Add Test Manager's modules path +g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.append(g_ksTestManagerDir) + +# Test Manager imports +from testmanager import config; +from testmanager.core.db import TMDatabaseConnection; +from testmanager.core.testset import TestSetLogic; + + +class ArchiveDelFilesBatchJob(object): # pylint: disable=too-few-public-methods + """ + Log+files comp + """ + + def __init__(self, oOptions): + """ + Parse command line + """ + self.fDryRun = oOptions.fDryRun; + self.fVerbose = oOptions.fVerbose; + self.sTempDir = tempfile.gettempdir(); + + self.dprint('Connecting to DB ...'); + self.oTestSetLogic = TestSetLogic(TMDatabaseConnection(self.dprint if self.fVerbose else None)); + + ## Fetches (and handles) all testsets up to this age (in hours). + self.uHoursAgeToHandle = 24; + ## Always remove files with these extensions. + self.asRemoveFileExt = [ 'webm' ]; + ## Always remove files which are bigger than this limit. + # Set to 0 to disable. + self.cbRemoveBiggerThan = 128 * 1024 * 1024; + + def dprint(self, sText): + """ Verbose output. """ + if self.fVerbose: + print(sText); + return True; + + def warning(self, sText): + """Prints a warning.""" + print(sText); + return True; + + def _replaceFile(self, sDstFile, sSrcFile, fDryRun = False, fForce = False): + """ + Replaces / moves a file safely by backing up the existing destination file (if any). + + Returns success indicator. + """ + + fRc = True; + + # Rename the destination file first (if any). + sDstFileTmp = None; + if os.path.exists(sDstFile): + sDstFileTmp = sDstFile + ".bak"; + if os.path.exists(sDstFileTmp): + if not fForce: + print('Replace file: Warning: Temporary destination file "%s" already exists, skipping' % (sDstFileTmp,)); + fRc = False; + else: + try: + os.remove(sDstFileTmp); + except Exception as e: + print('Replace file: Error deleting old temporary destination file "%s": %s' % (sDstFileTmp, e)); + fRc = False; + try: + if not fDryRun: + shutil.move(sDstFile, sDstFileTmp); + except Exception as e: + print('Replace file: Error moving old destination file "%s" to temporary file "%s": %s' \ + % (sDstFile, sDstFileTmp, e)); + fRc = False; + + if not fRc: + return False; + + try: + if not fDryRun: + shutil.move(sSrcFile, sDstFile); + except Exception as e: + print('Replace file: Error moving source file "%s" to destination "%s": %s' % (sSrcFile, sDstFile, e,)); + fRc = False; + + if sDstFileTmp: + if fRc: # Move succeeded, remove backup. + try: + if not fDryRun: + os.remove(sDstFileTmp); + except Exception as e: + print('Replace file: Error deleting temporary destination file "%s": %s' % (sDstFileTmp, e)); + fRc = False; + else: # Final move failed, roll back. + try: + if not fDryRun: + shutil.move(sDstFileTmp, sDstFile); + except Exception as e: + print('Replace file: Error restoring old destination file "%s": %s' % (sDstFile, e)); + fRc = False; + return fRc; + + def _processTestSetZip(self, idTestSet, sSrcZipFileAbs): + """ + Worker for processOneTestSet, which processes the testset's ZIP file. + + Returns success indicator. + """ + _ = idTestSet + + with tempfile.NamedTemporaryFile(dir=self.sTempDir, delete=False) as tmpfile: + sDstZipFileAbs = tmpfile.name; + + fRc = True; + + try: + oSrcZipFile = zipfile.ZipFile(sSrcZipFileAbs, 'r'); # pylint: disable=consider-using-with + self.dprint('Processing ZIP archive "%s" ...' % (sSrcZipFileAbs)); + try: + if not self.fDryRun: + oDstZipFile = zipfile.ZipFile(sDstZipFileAbs, 'w'); # pylint: disable=consider-using-with + self.dprint('Using temporary ZIP archive "%s"' % (sDstZipFileAbs)); + try: + # + # First pass: Gather information if we need to do some re-packing. + # + fDoRepack = False; + aoFilesToRepack = []; + for oCurFile in oSrcZipFile.infolist(): + self.dprint('Handling File "%s" ...' % (oCurFile.filename)) + sFileExt = os.path.splitext(oCurFile.filename)[1]; + + if sFileExt \ + and sFileExt[1:] in self.asRemoveFileExt: + self.dprint('\tMatches excluded extensions') + fDoRepack = True; + elif self.cbRemoveBiggerThan \ + and oCurFile.file_size > self.cbRemoveBiggerThan: + self.dprint('\tIs bigger than %d bytes (%d bytes)' % (self.cbRemoveBiggerThan, oCurFile.file_size)) + fDoRepack = True; + else: + aoFilesToRepack.append(oCurFile); + + if not fDoRepack: + oSrcZipFile.close(); + self.dprint('No re-packing necessary, skipping ZIP archive'); + return True; + + # + # Second pass: Re-pack all needed files into our temporary ZIP archive. + # + for oCurFile in aoFilesToRepack: + self.dprint('Re-packing file "%s"' % (oCurFile.filename,)) + if not self.fDryRun: + oBuf = oSrcZipFile.read(oCurFile); + oDstZipFile.writestr(oCurFile, oBuf); + + if not self.fDryRun: + oDstZipFile.close(); + + except Exception as oXcpt4: + print('Error handling file "%s" of archive "%s": %s' % (oCurFile.filename, sSrcZipFileAbs, oXcpt4,)); + return False; + + oSrcZipFile.close(); + + if fRc: + self.dprint('Moving file "%s" to "%s"' % (sDstZipFileAbs, sSrcZipFileAbs)); + fRc = self._replaceFile(sSrcZipFileAbs, sDstZipFileAbs, self.fDryRun); + + except Exception as oXcpt3: + print('Error creating temporary ZIP archive "%s": %s' % (sDstZipFileAbs, oXcpt3,)); + return False; + + except Exception as oXcpt1: + # Construct a meaningful error message. + if os.path.exists(sSrcZipFileAbs): + print('Error: Opening file "%s" failed: %s' % (sSrcZipFileAbs, oXcpt1)); + else: + print('Error: File "%s" not found.' % (sSrcZipFileAbs,)); + return False; + + return fRc; + + + def processOneTestSet(self, idTestSet, sBasename): + """ + Processes one single testset. + + Returns success indicator. + """ + + fRc = True; + self.dprint('Processing testset %d' % (idTestSet,)); + + # Construct absolute ZIP file path. + # ZIP is hardcoded in config, so do here. + sSrcZipFileAbs = os.path.join(config.g_ksZipFileAreaRootDir, sBasename + '.zip'); + + if self._processTestSetZip(idTestSet, sSrcZipFileAbs) is not True: + fRc = False; + + return fRc; + + def processTestSets(self): + """ + Processes all testsets according to the set configuration. + + Returns success indicator. + """ + + aoTestSets = self.oTestSetLogic.fetchByAge(cHoursBack = self.uHoursAgeToHandle); + cTestSets = len(aoTestSets); + print('Found %d entries in DB' % cTestSets); + if not cTestSets: + return True; # Nothing to do (yet). + + fRc = True; + for oTestSet in aoTestSets: + fRc = self.processOneTestSet(oTestSet.idTestSet, oTestSet.sBaseFilename) and fRc; + # Keep going. + + return fRc; + + @staticmethod + def main(): + """ C-style main(). """ + # + # Parse options. + # + + oParser = OptionParser(); + + # Generic options. + oParser.add_option('-v', '--verbose', dest = 'fVerbose', action = 'store_true', default = False, + help = 'Verbose output.'); + oParser.add_option('-q', '--quiet', dest = 'fVerbose', action = 'store_false', default = False, + help = 'Quiet operation.'); + oParser.add_option('-d', '--dry-run', dest = 'fDryRun', action = 'store_true', default = False, + help = 'Dry run, do not make any changes.'); + + (oOptions, asArgs) = oParser.parse_args(sys.argv[1:]); + if asArgs != []: + oParser.print_help(); + return 1; + + if oOptions.fDryRun: + print('***********************************'); + print('*** DRY RUN - NO FILES MODIFIED ***'); + print('***********************************'); + + # + # Do the work. + # + fRc = False; + + oBatchJob = ArchiveDelFilesBatchJob(oOptions); + fRc = oBatchJob.processTestSets(); + + if oOptions.fVerbose: + print('SUCCESS' if fRc else 'FAILURE'); + + return 0 if fRc is True else 1; + +if __name__ == '__main__': + sys.exit(ArchiveDelFilesBatchJob.main()); diff --git a/src/VBox/ValidationKit/testmanager/batch/regen_sched_queues.py b/src/VBox/ValidationKit/testmanager/batch/regen_sched_queues.py new file mode 100755 index 00000000..a20d6cbe --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/batch/regen_sched_queues.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# $Id: regen_sched_queues.py $ +# pylint: disable=line-too-long + +""" +Interface used by the admin to regenerate scheduling queues. +""" + +from __future__ import print_function; + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see . + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + +# Standard python imports +import sys; +import os; +from optparse import OptionParser; # pylint: disable=deprecated-module + +# Add Test Manager's modules path +g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))); +sys.path.append(g_ksTestManagerDir); + +# Test Manager imports +from testmanager.core.db import TMDatabaseConnection; +from testmanager.core.schedulerbase import SchedulerBase; +from testmanager.core.schedgroup import SchedGroupLogic; + + + +class RegenSchedQueues(object): # pylint: disable=too-few-public-methods + """ + Regenerates all the scheduling queues. + """ + + def __init__(self): + """ + Parse command line. + """ + + oParser = OptionParser(); + oParser.add_option('-q', '--quiet', dest = 'fQuiet', action = 'store_true', default = False, + help = 'Quiet execution'); + oParser.add_option('-u', '--uid', dest = 'uid', action = 'store', type = 'int', default = 1, + help = 'User ID to accredit with this job'); + oParser.add_option('--profile', dest = 'fProfile', action = 'store_true', default = False, + help = 'User ID to accredit with this job'); + + (self.oConfig, _) = oParser.parse_args(); + + + def doIt(self): + """ + Does the job. + """ + oDb = TMDatabaseConnection(); + + aoGroups = SchedGroupLogic(oDb).getAll(); + iRc = 0; + for oGroup in aoGroups: + if not self.oConfig.fQuiet: + print('%s (ID %#d):' % (oGroup.sName, oGroup.idSchedGroup,)); + try: + (aoErrors, asMessages) = SchedulerBase.recreateQueue(oDb, self.oConfig.uid, oGroup.idSchedGroup, 2); + except Exception as oXcpt: + oDb.rollback(); + print(' !!Hit exception processing "%s": %s' % (oGroup.sName, oXcpt,)); + else: + if not aoErrors: + if not self.oConfig.fQuiet: + print(' Successfully regenerated.'); + else: + iRc = 1; + print(' %d errors:' % (len(aoErrors,))); + for oError in aoErrors: + if oError[1] is None: + print(' !!%s' % (oError[0],)); + else: + print(' !!%s (%s)' % (oError[0], oError[1])); + if asMessages and not self.oConfig.fQuiet: + print(' %d messages:' % (len(asMessages),)); + for sMsg in asMessages: + print(' ##%s' % (sMsg,)); + return iRc; + + @staticmethod + def main(): + """ Main function. """ + oMain = RegenSchedQueues(); + if oMain.oConfig.fProfile is not True: + iRc = oMain.doIt(); + else: + import cProfile; + oProfiler = cProfile.Profile(); + iRc = oProfiler.runcall(oMain.doIt); + oProfiler.print_stats(sort = 'time'); + oProfiler = None; + return iRc; + +if __name__ == '__main__': + sys.exit(RegenSchedQueues().main()); + diff --git a/src/VBox/ValidationKit/testmanager/batch/vcs_import.py b/src/VBox/ValidationKit/testmanager/batch/vcs_import.py new file mode 100755 index 00000000..bffa576b --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/batch/vcs_import.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# $Id: vcs_import.py $ + +""" +Cron job for importing revision history for a repository. +""" + +from __future__ import print_function; + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see . + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + +# Standard python imports +import sys; +import os; +from optparse import OptionParser; # pylint: disable=deprecated-module +import xml.etree.ElementTree as ET; + +# Add Test Manager's modules path +g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))); +sys.path.append(g_ksTestManagerDir); + +# Test Manager imports +from testmanager.config import g_kdBugTrackers; +from testmanager.core.db import TMDatabaseConnection; +from testmanager.core.vcsrevisions import VcsRevisionData, VcsRevisionLogic; +from testmanager.core.vcsbugreference import VcsBugReferenceData, VcsBugReferenceLogic; +from common import utils; + +# Python 3 hacks: +if sys.version_info[0] >= 3: + long = int; # pylint: disable=redefined-builtin,invalid-name + + +class VcsImport(object): # pylint: disable=too-few-public-methods + """ + Imports revision history from a VSC into the Test Manager database. + """ + + class BugTracker(object): + def __init__(self, sDbName, sTag): + self.sDbName = sDbName; + self.sTag = sTag; + + + def __init__(self): + """ + Parse command line. + """ + + oParser = OptionParser() + oParser.add_option('-b', '--only-bug-refs', dest = 'fBugRefsOnly', action = 'store_true', + help = 'Only do bug references, not revisions.'); + oParser.add_option('-e', '--extra-option', dest = 'asExtraOptions', metavar = 'vcsoption', action = 'append', + help = 'Adds a extra option to the command retrieving the log.'); + oParser.add_option('-f', '--full', dest = 'fFull', action = 'store_true', + help = 'Full revision history import.'); + oParser.add_option('-q', '--quiet', dest = 'fQuiet', action = 'store_true', + help = 'Quiet execution'); + oParser.add_option('-R', '--repository', dest = 'sRepository', metavar = '', + help = 'Version control repository name.'); + oParser.add_option('-s', '--start-revision', dest = 'iStartRevision', metavar = 'start-revision', + type = "int", default = 0, + help = 'The revision to start at when doing a full import.'); + oParser.add_option('-t', '--type', dest = 'sType', metavar = '', + help = 'The VCS type (default: svn)', choices = [ 'svn', ], default = 'svn'); + oParser.add_option('-u', '--url', dest = 'sUrl', metavar = '', + help = 'The VCS URL'); + + (self.oConfig, _) = oParser.parse_args(); + + # Check command line + asMissing = []; + if self.oConfig.sUrl is None: asMissing.append('--url'); + if self.oConfig.sRepository is None: asMissing.append('--repository'); + if asMissing: + sys.stderr.write('syntax error: Missing: %s\n' % (asMissing,)); + sys.exit(1); + + assert self.oConfig.sType == 'svn'; + + def main(self): + """ + Main function. + """ + oDb = TMDatabaseConnection(); + oLogic = VcsRevisionLogic(oDb); + oBugLogic = VcsBugReferenceLogic(oDb); + + # Where to start. + iStartRev = 0; + if not self.oConfig.fFull: + if not self.oConfig.fBugRefsOnly: + iStartRev = oLogic.getLastRevision(self.oConfig.sRepository); + else: + iStartRev = oBugLogic.getLastRevision(self.oConfig.sRepository); + if iStartRev == 0: + iStartRev = self.oConfig.iStartRevision; + + # Construct a command line. + os.environ['LC_ALL'] = 'en_US.utf-8'; + asArgs = [ + 'svn', + 'log', + '--xml', + '--revision', str(iStartRev) + ':HEAD', + ]; + if self.oConfig.asExtraOptions is not None: + asArgs.extend(self.oConfig.asExtraOptions); + asArgs.append(self.oConfig.sUrl); + if not self.oConfig.fQuiet: + print('Executing: %s' % (asArgs,)); + sLogXml = utils.processOutputChecked(asArgs); + + # Parse the XML and add the entries to the database. + oParser = ET.XMLParser(target = ET.TreeBuilder(), encoding = 'utf-8'); + oParser.feed(sLogXml.encode('utf-8')); # Does its own decoding; processOutputChecked always gives us decoded utf-8 now. + oRoot = oParser.close(); + + for oLogEntry in oRoot.findall('logentry'): + iRevision = int(oLogEntry.get('revision')); + sAuthor = oLogEntry.findtext('author', 'unspecified').strip(); # cvs2svn entries doesn't have an author. + sDate = oLogEntry.findtext('date').strip(); + sRawMsg = oLogEntry.findtext('msg', '').strip(); + sMessage = sRawMsg; + if sMessage == '': + sMessage = ' '; + elif len(sMessage) > VcsRevisionData.kcchMax_sMessage: + sMessage = sMessage[:VcsRevisionData.kcchMax_sMessage - 4] + ' ...'; + if not self.oConfig.fQuiet: + utils.printOut(u'sDate=%s iRev=%u sAuthor=%s sMsg[%s]=%s' + % (sDate, iRevision, sAuthor, type(sMessage).__name__, sMessage)); + + if not self.oConfig.fBugRefsOnly: + oData = VcsRevisionData().initFromValues(self.oConfig.sRepository, iRevision, sDate, sAuthor, sMessage); + oLogic.addVcsRevision(oData); + + # Analyze the raw message looking for bug tracker references. + for oBugTracker in g_kdBugTrackers.values(): + for sTag in oBugTracker.asCommitTags: + off = sRawMsg.find(sTag); + while off >= 0: + off += len(sTag); + while off < len(sRawMsg) and sRawMsg[off].isspace(): + off += 1; + + if off < len(sRawMsg) and sRawMsg[off].isdigit(): + offNum = off; + while off < len(sRawMsg) and sRawMsg[off].isdigit(): + off += 1; + try: + iBugNo = long(sRawMsg[offNum:off]); + except Exception as oXcpt: + utils.printErr(u'error! exception(r%s,"%s"): -> %s' % (iRevision, sRawMsg[offNum:off], oXcpt,)); + else: + if not self.oConfig.fQuiet: + utils.printOut(u' r%u -> sBugTracker=%s iBugNo=%s' + % (iRevision, oBugTracker.sDbId, iBugNo,)); + + oBugData = VcsBugReferenceData().initFromValues(self.oConfig.sRepository, iRevision, + oBugTracker.sDbId, iBugNo); + oBugLogic.addVcsBugReference(oBugData); + + # next + off = sRawMsg.find(sTag, off); + + oDb.commit(); + + oDb.close(); + return 0; + +if __name__ == '__main__': + sys.exit(VcsImport().main()); + diff --git a/src/VBox/ValidationKit/testmanager/batch/virtual_test_sheriff.py b/src/VBox/ValidationKit/testmanager/batch/virtual_test_sheriff.py new file mode 100755 index 00000000..51999e21 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/batch/virtual_test_sheriff.py @@ -0,0 +1,1832 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# $Id: virtual_test_sheriff.py $ +# pylint: disable=line-too-long + +""" +Virtual Test Sheriff. + +Duties: + - Try to a assign failure reasons to recently failed tests. + - Reboot or disable bad test boxes. + +""" + +from __future__ import print_function; + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see . + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Standard python imports +import hashlib; +import os; +import re; +import smtplib; +#import subprocess; +import sys; +from email.mime.multipart import MIMEMultipart; +from email.mime.text import MIMEText; +from email.utils import COMMASPACE; + +if sys.version_info[0] >= 3: + from io import BytesIO as BytesIO; # pylint: disable=import-error,no-name-in-module,useless-import-alias +else: + from StringIO import StringIO as BytesIO; # pylint: disable=import-error,no-name-in-module,useless-import-alias +from optparse import OptionParser; # pylint: disable=deprecated-module +from PIL import Image; # pylint: disable=import-error + +# Add Test Manager's modules path +g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))); +sys.path.append(g_ksTestManagerDir); + +# Test Manager imports +from common import utils; +from testmanager.core.db import TMDatabaseConnection; +from testmanager.core.build import BuildDataEx; +from testmanager.core.failurereason import FailureReasonLogic; +from testmanager.core.testbox import TestBoxLogic, TestBoxData; +from testmanager.core.testcase import TestCaseDataEx; +from testmanager.core.testgroup import TestGroupData; +from testmanager.core.testset import TestSetLogic, TestSetData; +from testmanager.core.testresults import TestResultLogic, TestResultFileData; +from testmanager.core.testresultfailures import TestResultFailureLogic, TestResultFailureData; +from testmanager.core.useraccount import UserAccountLogic; +from testmanager.config import g_ksSmtpHost, g_kcSmtpPort, g_ksAlertFrom, \ + g_ksAlertSubject, g_asAlertList #, g_ksLomPassword; + +# Python 3 hacks: +if sys.version_info[0] >= 3: + xrange = range; # pylint: disable=redefined-builtin,invalid-name + + +class VirtualTestSheriffCaseFile(object): + """ + A failure investigation case file. + + """ + + + ## Max log file we'll read into memory. (256 MB) + kcbMaxLogRead = 0x10000000; + + def __init__(self, oSheriff, oTestSet, oTree, oBuild, oTestBox, oTestGroup, oTestCase): + self.oSheriff = oSheriff; + self.oTestSet = oTestSet; # TestSetData + self.oTree = oTree; # TestResultDataEx + self.oBuild = oBuild; # BuildDataEx + self.oTestBox = oTestBox; # TestBoxData + self.oTestGroup = oTestGroup; # TestGroupData + self.oTestCase = oTestCase; # TestCaseDataEx + self.sMainLog = ''; # The main log file. Empty string if not accessible. + self.sSvcLog = ''; # The VBoxSVC log file. Empty string if not accessible. + + # Generate a case file name. + self.sName = '#%u: %s' % (self.oTestSet.idTestSet, self.oTestCase.sName,) + self.sLongName = '#%u: "%s" on "%s" running %s %s (%s), "%s" by %s, using %s %s %s r%u' \ + % ( self.oTestSet.idTestSet, + self.oTestCase.sName, + self.oTestBox.sName, + self.oTestBox.sOs, + self.oTestBox.sOsVersion, + self.oTestBox.sCpuArch, + self.oTestBox.sCpuName, + self.oTestBox.sCpuVendor, + self.oBuild.oCat.sProduct, + self.oBuild.oCat.sBranch, + self.oBuild.oCat.sType, + self.oBuild.iRevision, ); + + # Investigation notes. + self.tReason = None; # None or one of the ktReason_XXX constants. + self.dReasonForResultId = {}; # Reason assignments indexed by idTestResult. + self.dCommentForResultId = {}; # Comment assignments indexed by idTestResult. + + # + # Reason. + # + + def noteReason(self, tReason): + """ Notes down a possible reason. """ + self.oSheriff.dprint(u'noteReason: %s -> %s' % (self.tReason, tReason,)); + self.tReason = tReason; + return True; + + def noteReasonForId(self, tReason, idTestResult, sComment = None): + """ Notes down a possible reason for a specific test result. """ + self.oSheriff.dprint(u'noteReasonForId: %u: %s -> %s%s' + % (idTestResult, self.dReasonForResultId.get(idTestResult, None), tReason, + (u' (%s)' % (sComment,)) if sComment is not None else '')); + self.dReasonForResultId[idTestResult] = tReason; + if sComment is not None: + self.dCommentForResultId[idTestResult] = sComment; + return True; + + + # + # Test classification. + # + + def isVBoxTest(self): + """ Test classification: VirtualBox (using the build) """ + return self.oBuild.oCat.sProduct.lower() in [ 'virtualbox', 'vbox' ]; + + def isVBoxUnitTest(self): + """ Test case classification: The unit test doing all our testcase/*.cpp stuff. """ + return self.isVBoxTest() \ + and (self.oTestCase.sName.lower() == 'unit tests' or self.oTestCase.sName.lower().startswith('misc: unit tests')); + + def isVBoxInstallTest(self): + """ Test case classification: VirtualBox Guest installation test. """ + return self.isVBoxTest() \ + and self.oTestCase.sName.lower().startswith('install:'); + + def isVBoxUnattendedInstallTest(self): + """ Test case classification: VirtualBox Guest installation test. """ + return self.isVBoxTest() \ + and self.oTestCase.sName.lower().startswith('uinstall:'); + + def isVBoxUSBTest(self): + """ Test case classification: VirtualBox USB test. """ + return self.isVBoxTest() \ + and self.oTestCase.sName.lower().startswith('usb:'); + + def isVBoxStorageTest(self): + """ Test case classification: VirtualBox Storage test. """ + return self.isVBoxTest() \ + and self.oTestCase.sName.lower().startswith('storage:'); + + def isVBoxGAsTest(self): + """ Test case classification: VirtualBox Guest Additions test. """ + return self.isVBoxTest() \ + and ( self.oTestCase.sName.lower().startswith('guest additions') + or self.oTestCase.sName.lower().startswith('ga\'s tests')); + + def isVBoxAPITest(self): + """ Test case classification: VirtualBox API test. """ + return self.isVBoxTest() \ + and self.oTestCase.sName.lower().startswith('api:'); + + def isVBoxBenchmarkTest(self): + """ Test case classification: VirtualBox Benchmark test. """ + return self.isVBoxTest() \ + and self.oTestCase.sName.lower().startswith('benchmark:'); + + def isVBoxSmokeTest(self): + """ Test case classification: Smoke test. """ + return self.isVBoxTest() \ + and self.oTestCase.sName.lower().startswith('smoketest'); + + def isVBoxSerialTest(self): + """ Test case classification: Smoke test. """ + return self.isVBoxTest() \ + and self.oTestCase.sName.lower().startswith('serial:'); + + + # + # Utility methods. + # + + def getMainLog(self): + """ + Tries to read the main log file since this will be the first source of information. + """ + if self.sMainLog: + return self.sMainLog; + (oFile, oSizeOrError, _) = self.oTestSet.openFile('main.log', 'rb'); + if oFile is not None: + try: + self.sMainLog = oFile.read(min(self.kcbMaxLogRead, oSizeOrError)).decode('utf-8', 'replace'); + except Exception as oXcpt: + self.oSheriff.vprint(u'Error reading main log file: %s' % (oXcpt,)) + self.sMainLog = ''; + else: + self.oSheriff.vprint(u'Error opening main log file: %s' % (oSizeOrError,)); + return self.sMainLog; + + def getLogFile(self, oFile): + """ + Tries to read the given file as a utf-8 log file. + oFile is a TestFileDataEx instance. + Returns empty string if problems opening or reading the file. + """ + sContent = ''; + (oFile, oSizeOrError, _) = self.oTestSet.openFile(oFile.sFile, 'rb'); + if oFile is not None: + try: + sContent = oFile.read(min(self.kcbMaxLogRead, oSizeOrError)).decode('utf-8', 'replace'); + except Exception as oXcpt: + self.oSheriff.vprint(u'Error reading the "%s" log file: %s' % (oFile.sFile, oXcpt,)) + else: + self.oSheriff.vprint(u'Error opening the "%s" log file: %s' % (oFile.sFile, oSizeOrError,)); + return sContent; + + def getSvcLog(self): + """ + Tries to read the VBoxSVC log file as it typically not associated with a failing test result. + Note! Returns the first VBoxSVC log file we find. + """ + if not self.sSvcLog: + aoSvcLogFiles = self.oTree.getListOfLogFilesByKind(TestResultFileData.ksKind_LogReleaseSvc); + if aoSvcLogFiles: + self.sSvcLog = self.getLogFile(aoSvcLogFiles[0]); + return self.sSvcLog; + + def getScreenshotSha256(self, oFile): + """ + Tries to read the given screenshot file, uncompress it, and do SHA-2 + on the raw pixels. + Returns SHA-2 digest string on success, None on failure. + """ + (oImgFile, _, _) = self.oTestSet.openFile(oFile.sFile, 'rb'); + try: + abImageFile = oImgFile.read(); + except Exception as oXcpt: + self.oSheriff.vprint(u'Error reading the "%s" image file: %s' % (oFile.sFile, oXcpt,)) + else: + try: + oImage = Image.open(BytesIO(abImageFile)); + except Exception as oXcpt: + self.oSheriff.vprint(u'Error opening the "%s" image bytes using PIL.Image.open: %s' % (oFile.sFile, oXcpt,)) + else: + try: + oHash = hashlib.sha256(); + if hasattr(oImage, 'tobytes'): + oHash.update(oImage.tobytes()); + else: + oHash.update(oImage.tostring()); # pylint: disable=no-member + except Exception as oXcpt: + self.oSheriff.vprint(u'Error hashing the uncompressed image bytes for "%s": %s' % (oFile.sFile, oXcpt,)) + else: + return oHash.hexdigest(); + return None; + + + + def isSingleTestFailure(self): + """ + Figure out if this is a single test failing or if it's one of the + more complicated ones. + """ + if self.oTree.cErrors == 1: + return True; + if self.oTree.deepCountErrorContributers() <= 1: + return True; + return False; + + + +class VirtualTestSheriff(object): # pylint: disable=too-few-public-methods + """ + Add build info into Test Manager database. + """ + + ## The user account for the virtual sheriff. + ksLoginName = 'vsheriff'; + + def __init__(self): + """ + Parse command line. + """ + self.oDb = None; + self.tsNow = None; + self.oTestResultLogic = None; + self.oTestSetLogic = None; + self.oFailureReasonLogic = None; # FailureReasonLogic; + self.oTestResultFailureLogic = None; # TestResultFailureLogic + self.oLogin = None; + self.uidSelf = -1; + self.oLogFile = None; + self.asBsodReasons = []; + self.asUnitTestReasons = []; + + oParser = OptionParser(); + oParser.add_option('--start-hours-ago', dest = 'cStartHoursAgo', metavar = '', default = 0, type = 'int', + help = 'When to start specified as hours relative to current time. Defauls is right now.', ); + oParser.add_option('--hours-period', dest = 'cHoursBack', metavar = '', default = 2, type = 'int', + help = 'Work period specified in hours. Defauls is 2 hours.'); + oParser.add_option('--real-run-back', dest = 'fRealRun', action = 'store_true', default = False, + help = 'Whether to commit the findings to the database. Default is a dry run.'); + oParser.add_option('--testset', dest = 'aidTestSets', metavar = '', default = [], type = 'int', action = 'append', + help = 'Only investigate this one. Accumulates IDs when repeated.'); + oParser.add_option('-q', '--quiet', dest = 'fQuiet', action = 'store_true', default = False, + help = 'Quiet execution'); + oParser.add_option('-l', '--log', dest = 'sLogFile', metavar = '', default = None, + help = 'Where to log messages.'); + oParser.add_option('--debug', dest = 'fDebug', action = 'store_true', default = False, + help = 'Enables debug mode.'); + + (self.oConfig, _) = oParser.parse_args(); + + if self.oConfig.sLogFile: + self.oLogFile = open(self.oConfig.sLogFile, "a"); # pylint: disable=consider-using-with,unspecified-encoding + self.oLogFile.write('VirtualTestSheriff: $Revision: 155244 $ \n'); + + + def eprint(self, sText): + """ + Prints error messages. + Returns 1 (for exit code usage.) + """ + print('error: %s' % (sText,)); + if self.oLogFile is not None: + if sys.version_info[0] >= 3: + self.oLogFile.write(u'error: %s\n' % (sText,)); + else: + self.oLogFile.write((u'error: %s\n' % (sText,)).encode('utf-8')); + return 1; + + def dprint(self, sText): + """ + Prints debug info. + """ + if self.oConfig.fDebug: + if not self.oConfig.fQuiet: + print('debug: %s' % (sText, )); + if self.oLogFile is not None: + if sys.version_info[0] >= 3: + self.oLogFile.write(u'debug: %s\n' % (sText,)); + else: + self.oLogFile.write((u'debug: %s\n' % (sText,)).encode('utf-8')); + return 0; + + def vprint(self, sText): + """ + Prints verbose info. + """ + if not self.oConfig.fQuiet: + print('info: %s' % (sText,)); + if self.oLogFile is not None: + if sys.version_info[0] >= 3: + self.oLogFile.write(u'info: %s\n' % (sText,)); + else: + self.oLogFile.write((u'info: %s\n' % (sText,)).encode('utf-8')); + return 0; + + def getFailureReason(self, tReason): + """ Gets the failure reason object for tReason. """ + return self.oFailureReasonLogic.cachedLookupByNameAndCategory(tReason[1], tReason[0]); + + def selfCheck(self): + """ Does some self checks, looking up things we expect to be in the database and such. """ + rcExit = 0; + for sAttr in dir(self.__class__): + if sAttr.startswith('ktReason_'): + tReason = getattr(self.__class__, sAttr); + oFailureReason = self.getFailureReason(tReason); + if oFailureReason is None: + rcExit = self.eprint(u'Failed to find failure reason "%s" in category "%s" in the database!' + % (tReason[1], tReason[0],)); + + # Check the user account as well. + if self.oLogin is None: + oLogin = UserAccountLogic(self.oDb).tryFetchAccountByLoginName(VirtualTestSheriff.ksLoginName); + if oLogin is None: + rcExit = self.eprint(u'Cannot find my user account "%s"!' % (VirtualTestSheriff.ksLoginName,)); + return rcExit; + + def sendEmailAlert(self, uidAuthor, sBodyText): + """ + Sends email alert. + """ + + # Get author email + self.oDb.execute('SELECT sEmail FROM Users WHERE uid=%s', (uidAuthor,)); + sFrom = self.oDb.fetchOne(); + if sFrom is not None: + sFrom = sFrom[0]; + else: + sFrom = g_ksAlertFrom; + + # Gather recipient list. + asEmailList = []; + for sUser in g_asAlertList: + self.oDb.execute('SELECT sEmail FROM Users WHERE sUsername=%s', (sUser,)); + sEmail = self.oDb.fetchOne(); + if sEmail: + asEmailList.append(sEmail[0]); + if not asEmailList: + return self.eprint('No email addresses to send alter to!'); + + # Compose the message. + oMsg = MIMEMultipart(); + oMsg['From'] = sFrom; + oMsg['To'] = COMMASPACE.join(asEmailList); + oMsg['Subject'] = g_ksAlertSubject; + oMsg.attach(MIMEText(sBodyText, 'plain')) + + # Try send it. + try: + oSMTP = smtplib.SMTP(g_ksSmtpHost, g_kcSmtpPort); + oSMTP.sendmail(sFrom, asEmailList, oMsg.as_string()) + oSMTP.quit() + except smtplib.SMTPException as oXcpt: + return self.eprint('Failed to send mail: %s' % (oXcpt,)); + + return 0; + + def badTestBoxManagement(self): + """ + Looks for bad test boxes and first tries once to reboot them then disables them. + """ + rcExit = 0; + + # + # We skip this entirely if we're running in the past and not in harmless debug mode. + # + if self.oConfig.cStartHoursAgo != 0 \ + and (not self.oConfig.fDebug or self.oConfig.fRealRun): + return rcExit; + tsNow = self.tsNow if self.oConfig.fDebug else None; + cHoursBack = self.oConfig.cHoursBack if self.oConfig.fDebug else 2; + oTestBoxLogic = TestBoxLogic(self.oDb); + + # + # Generate a list of failures reasons we consider bad-testbox behavior. + # + aidFailureReasons = [ + self.getFailureReason(self.ktReason_Host_DriverNotLoaded).idFailureReason, + self.getFailureReason(self.ktReason_Host_DriverNotUnloading).idFailureReason, + self.getFailureReason(self.ktReason_Host_DriverNotCompilable).idFailureReason, + self.getFailureReason(self.ktReason_Host_InstallationFailed).idFailureReason, + ]; + + # + # Get list of bad test boxes for given period and check them out individually. + # + aidBadTestBoxes = self.oTestSetLogic.fetchBadTestBoxIds(cHoursBack = cHoursBack, tsNow = tsNow, + aidFailureReasons = aidFailureReasons); + for idTestBox in aidBadTestBoxes: + # Skip if the testbox is already disabled or has a pending reboot command. + try: + oTestBox = TestBoxData().initFromDbWithId(self.oDb, idTestBox); + except Exception as oXcpt: + rcExit = self.eprint('Failed to get data for test box #%u in badTestBoxManagement: %s' % (idTestBox, oXcpt,)); + continue; + if not oTestBox.fEnabled: + self.dprint(u'badTestBoxManagement: Skipping test box #%u (%s) as it has been disabled already.' + % ( idTestBox, oTestBox.sName, )); + continue; + if oTestBox.enmPendingCmd != TestBoxData.ksTestBoxCmd_None: + self.dprint(u'badTestBoxManagement: Skipping test box #%u (%s) as it has a command pending: %s' + % ( idTestBox, oTestBox.sName, oTestBox.enmPendingCmd)); + continue; + + # Get the most recent testsets for this box (descending on tsDone) and see how bad it is. + aoSets = self.oTestSetLogic.fetchSetsForTestBox(idTestBox, cHoursBack = cHoursBack, tsNow = tsNow); + cOkay = 0; + cBad = 0; + iFirstOkay = len(aoSets); + for iSet, oSet in enumerate(aoSets): + if oSet.enmStatus == TestSetData.ksTestStatus_BadTestBox: + cBad += 1; + else: + # Check for bad failure reasons. + oFailure = None; + if oSet.enmStatus in TestSetData.kasBadTestStatuses: + (oTree, _ ) = self.oTestResultLogic.fetchResultTree(oSet.idTestSet) + aoFailedResults = oTree.getListOfFailures(); + for oFailedResult in aoFailedResults: + oFailure = self.oTestResultFailureLogic.getById(oFailedResult.idTestResult); + if oFailure is not None and oFailure.idFailureReason in aidFailureReasons: + break; + oFailure = None; + if oFailure is not None: + cBad += 1; + else: + # This is an okay test result then. + ## @todo maybe check the elapsed time here, it could still be a bad run? + cOkay += 1; + iFirstOkay = min(iFirstOkay, iSet); + if iSet > 10: + break; + + # We react if there are two or more bad-testbox statuses at the head of the + # history and at least three in the last 10 results. + if iFirstOkay >= 2 and cBad > 2: + if oTestBoxLogic.hasTestBoxRecentlyBeenRebooted(idTestBox, cHoursBack = cHoursBack, tsNow = tsNow): + sComment = u'Disabling testbox #%u (%s) - iFirstOkay=%u cBad=%u cOkay=%u' \ + % (idTestBox, oTestBox.sName, iFirstOkay, cBad, cOkay); + self.vprint(sComment); + self.sendEmailAlert(self.uidSelf, sComment); + if self.oConfig.fRealRun is True: + try: + oTestBoxLogic.disableTestBox(idTestBox, self.uidSelf, fCommit = True, + sComment = 'Automatically disabled (iFirstOkay=%u cBad=%u cOkay=%u)' + % (iFirstOkay, cBad, cOkay),); + except Exception as oXcpt: + rcExit = self.eprint(u'Error disabling testbox #%u (%u): %s\n' % (idTestBox, oTestBox.sName, oXcpt,)); + else: + sComment = u'Rebooting testbox #%u (%s) - iFirstOkay=%u cBad=%u cOkay=%u' \ + % (idTestBox, oTestBox.sName, iFirstOkay, cBad, cOkay); + self.vprint(sComment); + self.sendEmailAlert(self.uidSelf, sComment); + if self.oConfig.fRealRun is True: + try: + oTestBoxLogic.rebootTestBox(idTestBox, self.uidSelf, fCommit = True, + sComment = 'Automatically rebooted (iFirstOkay=%u cBad=%u cOkay=%u)' + % (iFirstOkay, cBad, cOkay),); + except Exception as oXcpt: + rcExit = self.eprint(u'Error rebooting testbox #%u (%s): %s\n' % (idTestBox, oTestBox.sName, oXcpt,)); + else: + self.dprint(u'badTestBoxManagement: #%u (%s) looks ok: iFirstOkay=%u cBad=%u cOkay=%u' + % ( idTestBox, oTestBox.sName, iFirstOkay, cBad, cOkay)); + + ## @todo r=bird: review + rewrite; + ## - no selecting here, that belongs in the core/*.py files. + ## - preserve existing comments. + ## - doing way too much in the try/except block. + ## - No password quoting in the sshpass command that always fails (127). + ## - Timeout is way to low. testboxmem1 need more than 10 min to take a dump, ages to + ## get thru POST and another 5 just to time out in grub. Should be an hour or so. + ## Besides, it need to be constant elsewhere in the file, not a variable here. + ## + ## + ## Reset hanged testboxes + ## + #cStatusTimeoutMins = 10; + # + #self.oDb.execute('SELECT TestBoxStatuses.idTestBox\n' + # ' FROM TestBoxStatuses, TestBoxes\n' + # ' WHERE TestBoxStatuses.tsUpdated >= (CURRENT_TIMESTAMP - interval \'%s hours\')\n' + # ' AND TestBoxStatuses.tsUpdated < (CURRENT_TIMESTAMP - interval \'%s minutes\')\n' + # ' AND TestBoxStatuses.idTestBox = TestBoxes.idTestBox\n' + # ' AND Testboxes.tsExpire = \'infinity\'::timestamp', (cHoursBack,cStatusTimeoutMins)); + #for idTestBox in self.oDb.fetchAll(): + # idTestBox = idTestBox[0]; + # try: + # oTestBox = TestBoxData().initFromDbWithId(self.oDb, idTestBox); + # except Exception as oXcpt: + # rcExit = self.eprint('Failed to get data for test box #%u in badTestBoxManagement: %s' % (idTestBox, oXcpt,)); + # continue; + # # Skip if the testbox is already disabled, already reset or there's no iLOM + # if not oTestBox.fEnabled or oTestBox.ipLom is None or oTestBox.sComment is not None and oTestBox.sComment.find('Automatically reset') >= 0: + # self.dprint(u'badTestBoxManagement: Skipping test box #%u (%s) as it has been disabled already.' + # % ( idTestBox, oTestBox.sName, )); + # continue; + # ## @todo get iLOM credentials from a table? + # sCmd = 'sshpass -p%s ssh -oStrictHostKeyChecking=no root@%s show /SP && reset /SYS' % (g_ksLomPassword, oTestBox.ipLom,); + # try: + # oPs = subprocess.Popen(sCmd, stdout=subprocess.PIPE, shell=True); + # sStdout = oPs.communicate()[0]; + # iRC = oPs.wait(); + # + # oTestBox.sComment = 'Automatically reset (iRC=%u sStdout=%s)' % (iRC, sStdout,); + # oTestBoxLogic.editEntry(oTestBox, self.uidSelf, fCommit = True); + # + # sComment = u'Reset testbox #%u (%s) - iRC=%u sStduot=%s' % ( idTestBox, oTestBox.sName, iRC, sStdout); + # self.vprint(sComment); + # self.sendEmailAlert(self.uidSelf, sComment); + # + # except Exception as oXcpt: + # rcExit = self.eprint(u'Error resetting testbox #%u (%s): %s\n' % (idTestBox, oTestBox.sName, oXcpt,)); + # + return rcExit; + + + ## @name Failure reasons we know. + ## @{ + + ktReason_Add_Installer_Win_Failed = ( 'Additions', 'Win GA install' ); + ktReason_Add_ShFl_Automount = ( 'Additions', 'Automounting' ); + ktReason_Add_ShFl_FsPerf = ( 'Additions', 'FsPerf' ); + ktReason_Add_ShFl_FsPerf_Abend = ( 'Additions', 'FsPerf abend' ); + ktReason_Add_GstCtl_Preparations = ( 'Additions', 'GstCtl preparations' ); + ktReason_Add_GstCtl_SessionBasics = ( 'Additions', 'Session basics' ); + ktReason_Add_GstCtl_SessionProcRefs = ( 'Additions', 'Session process' ); + ktReason_Add_GstCtl_Session_Reboot = ( 'Additions', 'Session reboot' ); + ktReason_Add_GstCtl_CopyFromGuest_Timeout = ( 'Additions', 'CopyFromGuest timeout' ); + ktReason_Add_GstCtl_CopyToGuest_Timeout = ( 'Additions', 'CopyToGuest timeout' ); + ktReason_Add_GstCtl_CopyToGuest_DstEmpty = ( 'Additions', 'CopyToGuest dst empty' ); + ktReason_Add_GstCtl_CopyToGuest_DstExists = ( 'Additions', 'CopyToGuest dst exists' ); + ktReason_Add_FlushViewOfFile = ( 'Additions', 'FlushViewOfFile' ); + ktReason_Add_Mmap_Coherency = ( 'Additions', 'mmap coherency' ); + ktReason_BSOD_Recovery = ( 'BSOD', 'Recovery' ); + ktReason_BSOD_Automatic_Repair = ( 'BSOD', 'Automatic Repair' ); + ktReason_BSOD_0000007F = ( 'BSOD', '0x0000007F' ); + ktReason_BSOD_000000D1 = ( 'BSOD', '0x000000D1' ); + ktReason_BSOD_C0000225 = ( 'BSOD', '0xC0000225 (boot)' ); + ktReason_Guru_Generic = ( 'Guru Meditations', 'Generic Guru Meditation' ); + ktReason_Guru_VERR_IEM_INSTR_NOT_IMPLEMENTED = ( 'Guru Meditations', 'VERR_IEM_INSTR_NOT_IMPLEMENTED' ); + ktReason_Guru_VERR_IEM_ASPECT_NOT_IMPLEMENTED = ( 'Guru Meditations', 'VERR_IEM_ASPECT_NOT_IMPLEMENTED' ); + ktReason_Guru_VERR_TRPM_DONT_PANIC = ( 'Guru Meditations', 'VERR_TRPM_DONT_PANIC' ); + ktReason_Guru_VERR_PGM_PHYS_PAGE_RESERVED = ( 'Guru Meditations', 'VERR_PGM_PHYS_PAGE_RESERVED' ); + ktReason_Guru_VERR_VMX_INVALID_GUEST_STATE = ( 'Guru Meditations', 'VERR_VMX_INVALID_GUEST_STATE' ); + ktReason_Guru_VINF_EM_TRIPLE_FAULT = ( 'Guru Meditations', 'VINF_EM_TRIPLE_FAULT' ); + ktReason_Host_HostMemoryLow = ( 'Host', 'HostMemoryLow' ); + ktReason_Host_DriverNotLoaded = ( 'Host', 'Driver not loaded' ); + ktReason_Host_DriverNotUnloading = ( 'Host', 'Driver not unloading' ); + ktReason_Host_DriverNotCompilable = ( 'Host', 'Driver not compilable' ); + ktReason_Host_InstallationFailed = ( 'Host', 'Installation failed' ); + ktReason_Host_InstallationWantReboot = ( 'Host', 'Installation want reboot' ); + ktReason_Host_InvalidPackage = ( 'Host', 'ERROR_INSTALL_PACKAGE_INVALID' ); + ktReason_Host_InstallSourceAbsent = ( 'Host', 'ERROR_INSTALL_SOURCE_ABSENT' ); + ktReason_Host_NotSignedWithBuildCert = ( 'Host', 'Not signed with build cert' ); + ktReason_Host_DiskFull = ( 'Host', 'Host disk full' ); + ktReason_Host_DoubleFreeHeap = ( 'Host', 'Double free or corruption' ); + ktReason_Host_LeftoverService = ( 'Host', 'Leftover service' ); + ktReason_Host_win32com_gen_py = ( 'Host', 'win32com.gen_py' ); + ktReason_Host_Reboot_OSX_Watchdog_Timeout = ( 'Host Reboot', 'OSX Watchdog Timeout' ); + ktReason_Host_Modprobe_Failed = ( 'Host', 'Modprobe failed' ); + ktReason_Host_Install_Hang = ( 'Host', 'Install hang' ); + ktReason_Host_NetworkMisconfiguration = ( 'Host', 'Network misconfiguration' ); + ktReason_Host_TSTInfo_Accuracy_OOR = ( 'Host', 'TSTInfo accuracy out of range' ); + ktReason_Networking_Nonexistent_host_nic = ( 'Networking', 'Nonexistent host networking interface' ); + ktReason_Networking_VERR_INTNET_FLT_IF_NOT_FOUND = ( 'Networking', 'VERR_INTNET_FLT_IF_NOT_FOUND' ); + ktReason_OSInstall_GRUB_hang = ( 'O/S Install', 'GRUB hang' ); + ktReason_OSInstall_Udev_hang = ( 'O/S Install', 'udev hang' ); + ktReason_OSInstall_Sata_no_BM = ( 'O/S Install', 'SATA busmaster bit not set' ); + ktReason_Panic_BootManagerC000000F = ( 'Panic', 'Hardware Changed' ); + ktReason_Panic_MP_BIOS_IO_APIC = ( 'Panic', 'MP-BIOS/IO-APIC' ); + ktReason_Panic_HugeMemory = ( 'Panic', 'Huge memory assertion' ); + ktReason_Panic_IOAPICDoesntWork = ( 'Panic', 'IO-APIC and timer does not work' ); + ktReason_Panic_TxUnitHang = ( 'Panic', 'Tx Unit Hang' ); + ktReason_API_std_bad_alloc = ( 'API / (XP)COM', 'std::bad_alloc' ); + ktReason_API_Digest_Mismatch = ( 'API / (XP)COM', 'Digest mismatch' ); + ktReason_API_MoveVM_SharingViolation = ( 'API / (XP)COM', 'MoveVM sharing violation' ); + ktReason_API_MoveVM_InvalidParameter = ( 'API / (XP)COM', 'MoveVM invalid parameter' ); + ktReason_API_Open_Session_Failed = ( 'API / (XP)COM', 'Open session failed' ); + ktReason_XPCOM_Exit_Minus_11 = ( 'API / (XP)COM', 'exit -11' ); + ktReason_XPCOM_VBoxSVC_Hang = ( 'API / (XP)COM', 'VBoxSVC hang' ); + ktReason_XPCOM_VBoxSVC_Hang_Plus_Heap_Corruption = ( 'API / (XP)COM', 'VBoxSVC hang + heap corruption' ); + ktReason_XPCOM_NS_ERROR_CALL_FAILED = ( 'API / (XP)COM', 'NS_ERROR_CALL_FAILED' ); + ktReason_BootManager_Image_corrupt = ( 'Unknown', 'BOOTMGR Image corrupt' ); + ktReason_Unknown_Heap_Corruption = ( 'Unknown', 'Heap corruption' ); + ktReason_Unknown_Reboot_Loop = ( 'Unknown', 'Reboot loop' ); + ktReason_Unknown_File_Not_Found = ( 'Unknown', 'File not found' ); + ktReason_Unknown_HalReturnToFirmware = ( 'Unknown', 'HalReturnToFirmware' ); + ktReason_Unknown_VM_Crash = ( 'Unknown', 'VM crash' ); + ktReason_Unknown_VM_Terminated = ( 'Unknown', 'VM terminated' ); + ktReason_Unknown_VM_Start_Error = ( 'Unknown', 'VM Start Error' ); + ktReason_Unknown_VM_Runtime_Error = ( 'Unknown', 'VM Runtime Error' ); + ktReason_VMM_kvm_lock_spinning = ( 'VMM', 'kvm_lock_spinning' ); + ktReason_Ignore_Buggy_Test_Driver = ( 'Ignore', 'Buggy test driver' ); + ktReason_Ignore_Stale_Files = ( 'Ignore', 'Stale files' ); + ktReason_Buggy_Build_Broken_Build = ( 'Broken Build', 'Buggy build' ); + ktReason_GuestBug_CompizVBoxQt = ( 'Guest Bug', 'Compiz + VirtualBox Qt GUI crash' ); + ## @} + + ## BSOD category. + ksBsodCategory = 'BSOD'; + ## Special reason indicating that the flesh and blood sheriff has work to do. + ksBsodAddNew = 'Add new BSOD'; + + ## Unit test category. + ksUnitTestCategory = 'Unit'; + ## Special reason indicating that the flesh and blood sheriff has work to do. + ksUnitTestAddNew = 'Add new'; + + ## Used for indica that we shouldn't report anything for this test result ID and + ## consider promoting the previous error to test set level if it's the only one. + ktHarmless = ( 'Probably', 'Caused by previous error' ); + + + def caseClosed(self, oCaseFile): + """ + Reports the findings in the case and closes it. + """ + # + # Log it and create a dReasonForReasultId we can use below. + # + dCommentForResultId = oCaseFile.dCommentForResultId; + if oCaseFile.dReasonForResultId: + # Must weed out ktHarmless. + dReasonForResultId = {}; + for idKey, tReason in oCaseFile.dReasonForResultId.items(): + if tReason is not self.ktHarmless: + dReasonForResultId[idKey] = tReason; + if not dReasonForResultId: + self.vprint(u'TODO: Closing %s without a real reason, only %s.' + % (oCaseFile.sName, oCaseFile.dReasonForResultId)); + return False; + + # Try promote to single reason. + atValues = dReasonForResultId.values(); + fSingleReason = True; + if len(dReasonForResultId) == 1 and next(iter(dReasonForResultId.keys())) != oCaseFile.oTestSet.idTestResult: + self.dprint(u'Promoting single reason to whole set: %s' % (next(iter(atValues)),)); + elif len(dReasonForResultId) > 1 and len(atValues) == list(atValues).count(next(iter(atValues))): + self.dprint(u'Merged %d reasons to a single one: %s' % (len(atValues), next(iter(atValues)))); + else: + fSingleReason = False; + if fSingleReason: + dReasonForResultId = { oCaseFile.oTestSet.idTestResult: next(iter(atValues)), }; + if dCommentForResultId: + dCommentForResultId = { oCaseFile.oTestSet.idTestResult: next(iter(dCommentForResultId.values())), }; + elif oCaseFile.tReason is not None: + dReasonForResultId = { oCaseFile.oTestSet.idTestResult: oCaseFile.tReason, }; + else: + self.vprint(u'Closing %s without a reason - this should not happen!' % (oCaseFile.sName,)); + return False; + + self.vprint(u'Closing %s with following reason%s: %s' + % ( oCaseFile.sName, 's' if len(dReasonForResultId) > 1 else '', dReasonForResultId, )); + + # + # Add the test failure reason record(s). + # + for idTestResult, tReason in dReasonForResultId.items(): + oFailureReason = self.getFailureReason(tReason); + if oFailureReason is not None: + sComment = 'Set by $Revision: 155244 $' # Handy for reverting later. + if idTestResult in dCommentForResultId: + sComment += ': ' + dCommentForResultId[idTestResult]; + + oAdd = TestResultFailureData(); + oAdd.initFromValues(idTestResult = idTestResult, + idFailureReason = oFailureReason.idFailureReason, + uidAuthor = self.uidSelf, + idTestSet = oCaseFile.oTestSet.idTestSet, + sComment = sComment,); + if self.oConfig.fRealRun: + try: + self.oTestResultFailureLogic.addEntry(oAdd, self.uidSelf, fCommit = True); + except Exception as oXcpt: + self.eprint(u'caseClosed: Exception "%s" while adding reason %s for %s' + % (oXcpt, oAdd, oCaseFile.sLongName,)); + else: + self.eprint(u'caseClosed: Cannot locate failure reason: %s / %s' % ( tReason[0], tReason[1],)); + return True; + + # + # Tools for assiting log parsing. + # + + @staticmethod + def matchFollowedByLines(sStr, off, asFollowingLines): + """ Worker for isThisFollowedByTheseLines. """ + + # Advance off to the end of the line. + off = sStr.find('\n', off); + if off < 0: + return False; + off += 1; + + # Match each string with the subsequent lines. + for iLine, sLine in enumerate(asFollowingLines): + offEnd = sStr.find('\n', off); + if offEnd < 0: + return iLine + 1 == len(asFollowingLines) and sStr.find(sLine, off) < 0; + if sLine and sStr.find(sLine, off, offEnd) < 0: + return False; + + # next line. + off = offEnd + 1; + + return True; + + @staticmethod + def isThisFollowedByTheseLines(sStr, sFirst, asFollowingLines): + """ + Looks for a line contining sFirst which is then followed by lines + with the strings in asFollowingLines. (No newline chars anywhere!) + Returns True / False. + """ + off = sStr.find(sFirst, 0); + while off >= 0: + if VirtualTestSheriff.matchFollowedByLines(sStr, off, asFollowingLines): + return True; + off = sStr.find(sFirst, off + 1); + return False; + + @staticmethod + def findAndReturnRestOfLine(sHaystack, sNeedle): + """ + Looks for sNeedle in sHaystack. + Returns The text following the needle up to the end of the line. + Returns None if not found. + """ + if sHaystack is None: + return None; + off = sHaystack.find(sNeedle); + if off < 0: + return None; + off += len(sNeedle) + offEol = sHaystack.find('\n', off); + if offEol < 0: + offEol = len(sHaystack); + return sHaystack[off:offEol] + + @staticmethod + def findInAnyAndReturnRestOfLine(asHaystacks, sNeedle): + """ + Looks for sNeedle in zeroe or more haystacks (asHaystack). + Returns The text following the first needed found up to the end of the line. + Returns None if not found. + """ + for sHaystack in asHaystacks: + sRet = VirtualTestSheriff.findAndReturnRestOfLine(sHaystack, sNeedle); + if sRet is not None: + return sRet; + return None; + + + # + # The investigative units. + # + + katSimpleInstallUninstallMainLogReasons = [ + # ( Whether to stop on hit, reason tuple, needle text. ) + ( False, ktReason_Host_LeftoverService, + 'SERVICE_NAME: vbox' ), + ( False, ktReason_Host_LeftoverService, + 'Seems installation was skipped. Old version lurking behind? Not the fault of this build/test run!'), + ]; + + kdatSimpleInstallUninstallMainLogReasonsPerOs = { + 'darwin': [ + # ( Whether to stop on hit, reason tuple, needle text. ) + ( True, ktReason_Host_DriverNotUnloading, + 'Can\'t remove kext org.virtualbox.kext.VBoxDrv; services failed to terminate - 0xe00002c7' ), + ], + 'linux': [ + # ( Whether to stop on hit, reason tuple, needle text. ) + ( True, ktReason_Host_DriverNotCompilable, + 'This system is not currently set up to build kernel modules' ), + ( True, ktReason_Host_DriverNotCompilable, + 'This system is currently not set up to build kernel modules' ), + ( True, ktReason_Host_InstallationFailed, + 'vboxdrv.sh: failed: Look at /var/log/vbox-install.log to find out what went wrong.' ), + ( True, ktReason_Host_DriverNotUnloading, + 'Cannot unload module vboxdrv'), + ], + 'solaris': [ + # ( Whether to stop on hit, reason tuple, needle text. ) + ( True, ktReason_Host_DriverNotUnloading, 'can\'t unload the module: Device busy' ), + ( True, ktReason_Host_DriverNotUnloading, 'Unloading: Host module ...FAILED!' ), + ( True, ktReason_Host_DriverNotUnloading, 'Unloading: NetFilter (Crossbow) module ...FAILED!' ), + ( True, ktReason_Host_InstallationFailed, 'svcadm: Couldn\'t bind to svc.configd.' ), + ( True, ktReason_Host_InstallationFailed, 'pkgadd: ERROR: postinstall script did not complete successfully' ), + ], + 'win': [ + # ( Whether to stop on hit, reason tuple, needle text. ) + ( True, ktReason_Host_InstallationWantReboot, 'ERROR_SUCCESS_REBOOT_REQUIRED' ), + ( False, ktReason_Host_InstallationFailed, 'Installation error.' ), + ( True, ktReason_Host_InvalidPackage, 'Uninstaller failed, exit code: 1620' ), + ( True, ktReason_Host_InstallSourceAbsent, 'Uninstaller failed, exit code: 1612' ), + ], + }; + + + def investigateInstallUninstallFailure(self, oCaseFile, oFailedResult, sResultLog, fInstall): + """ + Investigates an install or uninstall failure. + + We lump the two together since the installation typically also performs + an uninstall first and will be seeing similar issues to the uninstall. + """ + self.dprint(u'%s + %s <<\n%s\n<<' % (oFailedResult.tsCreated, oFailedResult.tsElapsed, sResultLog,)); + + if fInstall and oFailedResult.enmStatus == TestSetData.ksTestStatus_TimedOut: + oCaseFile.noteReasonForId(self.ktReason_Host_Install_Hang, oFailedResult.idTestResult) + return True; + + atSimple = self.katSimpleInstallUninstallMainLogReasons; + if oCaseFile.oTestBox.sOs in self.kdatSimpleInstallUninstallMainLogReasonsPerOs: + atSimple = self.kdatSimpleInstallUninstallMainLogReasonsPerOs[oCaseFile.oTestBox.sOs] + atSimple; + + fFoundSomething = False; + for fStopOnHit, tReason, sNeedle in atSimple: + if sResultLog.find(sNeedle) > 0: + oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult); + if fStopOnHit: + return True; + fFoundSomething = True; + + return fFoundSomething if fFoundSomething else None; + + + def investigateBadTestBox(self, oCaseFile): + """ + Checks out bad-testbox statuses. + """ + _ = oCaseFile; + return False; + + + def investigateVBoxUnitTest(self, oCaseFile): + """ + Checks out a VBox unittest problem. + """ + + # + # Process simple test case failures first, using their name as reason. + # We do the reason management just like for BSODs. + # + cRelevantOnes = 0; + sMainLog = oCaseFile.getMainLog(); + aoFailedResults = oCaseFile.oTree.getListOfFailures(); + for oFailedResult in aoFailedResults: + if oFailedResult is oCaseFile.oTree: + self.vprint('TODO: toplevel failure'); + cRelevantOnes += 1 + + elif oFailedResult.sName == 'Installing VirtualBox': + sResultLog = TestSetData.extractLogSectionElapsed(sMainLog, oFailedResult.tsCreated, oFailedResult.tsElapsed); + self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = True) + cRelevantOnes += 1 + + elif oFailedResult.sName == 'Uninstalling VirtualBox': + sResultLog = TestSetData.extractLogSectionElapsed(sMainLog, oFailedResult.tsCreated, oFailedResult.tsElapsed); + self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = False) + cRelevantOnes += 1 + + elif oFailedResult.oParent is not None: + # Get the 2nd level node because that's where we'll find the unit test name. + while oFailedResult.oParent.oParent is not None: + oFailedResult = oFailedResult.oParent; + + # Only report a failure once. + if oFailedResult.idTestResult not in oCaseFile.dReasonForResultId: + sKey = oFailedResult.sName; + if sKey.startswith('testcase/'): + sKey = sKey[9:]; + if sKey in self.asUnitTestReasons: + tReason = ( self.ksUnitTestCategory, sKey ); + oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult); + else: + self.dprint(u'Unit test failure "%s" not found in %s;' % (sKey, self.asUnitTestReasons)); + tReason = ( self.ksUnitTestCategory, self.ksUnitTestAddNew ); + oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult, sComment = sKey); + cRelevantOnes += 1 + else: + self.vprint(u'Internal error: expected oParent to NOT be None for %s' % (oFailedResult,)); + + # + # If we've caught all the relevant ones by now, report the result. + # + if len(oCaseFile.dReasonForResultId) >= cRelevantOnes: + return self.caseClosed(oCaseFile); + return False; + + def extractGuestCpuStack(self, sInfoText): + """ + Extracts the guest CPU stacks from the input file. + + Returns a dictionary keyed by the CPU number, value being a list of + raw stack lines (no header). + Returns empty dictionary if no stacks where found. + """ + dRet = {}; + off = 0; + while True: + # Find the stack. + offStart = sInfoText.find('=== start guest stack VCPU ', off); + if offStart < 0: + break; + offEnd = sInfoText.find('=== end guest stack', offStart + 20); + if offEnd >= 0: + offEnd += 3; + else: + offEnd = sInfoText.find('=== start guest stack VCPU', offStart + 20); + if offEnd < 0: + offEnd = len(sInfoText); + + sStack = sInfoText[offStart : offEnd]; + sStack = sStack.replace('\r',''); # paranoia + asLines = sStack.split('\n'); + + # Figure the CPU. + asWords = asLines[0].split(); + if len(asWords) < 6 or not asWords[5].isdigit(): + break; + iCpu = int(asWords[5]); + + # Add it and advance. + dRet[iCpu] = [sLine.rstrip() for sLine in asLines[2:-1]] + off = offEnd; + return dRet; + + def investigateInfoKvmLockSpinning(self, oCaseFile, sInfoText, dLogs): + """ Investigates kvm_lock_spinning deadlocks """ + # + # Extract the stacks. We need more than one CPU to create a deadlock. + # + dStacks = self.extractGuestCpuStack(sInfoText); + self.dprint('kvm_lock_spinning: found %s stacks' % (len(dStacks),)); + if len(dStacks) >= 2: + # + # Examin each of the stacks. Each must have kvm_lock_spinning in + # one of the first three entries. + # + cHits = 0; + for asBacktrace in dStacks.values(): + for iFrame in xrange(min(3, len(asBacktrace))): + if asBacktrace[iFrame].find('kvm_lock_spinning') >= 0: + cHits += 1; + break; + self.dprint('kvm_lock_spinning: %s/%s hits' % (cHits, len(dStacks),)); + if cHits == len(dStacks): + return (True, self.ktReason_VMM_kvm_lock_spinning); + + _ = dLogs; _ = oCaseFile; + return (False, None); + + def investigateInfoHalReturnToFirmware(self, oCaseFile, sInfoText, dLogs): + """ Investigates HalReturnToFirmware hangs """ + del oCaseFile + del sInfoText + del dLogs + # hope that's sufficient + return (True, self.ktReason_Unknown_HalReturnToFirmware); + + ## Things we search a main or VM log for to figure out why something went bust. + ## @note DO NOT ADD MORE STUFF HERE! + ## Please use katSimpleMainLogReasons and katSimpleVmLogReasons instead! + katSimpleMainAndVmLogReasonsDeprecated = [ + # ( Whether to stop on hit, reason tuple, needle text. ) + ( False, ktReason_Guru_Generic, 'GuruMeditation' ), + ( False, ktReason_Guru_Generic, 'Guru Meditation' ), + ( True, ktReason_Guru_VERR_IEM_INSTR_NOT_IMPLEMENTED, 'VERR_IEM_INSTR_NOT_IMPLEMENTED' ), + ( True, ktReason_Guru_VERR_IEM_ASPECT_NOT_IMPLEMENTED, 'VERR_IEM_ASPECT_NOT_IMPLEMENTED' ), + ( True, ktReason_Guru_VERR_TRPM_DONT_PANIC, 'VERR_TRPM_DONT_PANIC' ), + ( True, ktReason_Guru_VERR_PGM_PHYS_PAGE_RESERVED, 'VERR_PGM_PHYS_PAGE_RESERVED' ), + ( True, ktReason_Guru_VERR_VMX_INVALID_GUEST_STATE, 'VERR_VMX_INVALID_GUEST_STATE' ), + ( True, ktReason_Guru_VINF_EM_TRIPLE_FAULT, 'VINF_EM_TRIPLE_FAULT' ), + ( True, ktReason_Networking_Nonexistent_host_nic, + 'rc=E_FAIL text="Nonexistent host networking interface, name \'eth0\' (VERR_INTERNAL_ERROR)"' ), + ( True, ktReason_Networking_VERR_INTNET_FLT_IF_NOT_FOUND, + 'Failed to attach the network LUN (VERR_INTNET_FLT_IF_NOT_FOUND)' ), + ( True, ktReason_Host_Reboot_OSX_Watchdog_Timeout, ': "OSX Watchdog Timeout: ' ), + ( False, ktReason_XPCOM_NS_ERROR_CALL_FAILED, + 'Exception: 0x800706be (Call to remote object failed (NS_ERROR_CALL_FAILED))' ), + ( True, ktReason_API_std_bad_alloc, 'Unexpected exception: std::bad_alloc' ), + ( True, ktReason_Host_HostMemoryLow, 'HostMemoryLow' ), + ( True, ktReason_Host_HostMemoryLow, 'Failed to procure handy pages; rc=VERR_NO_MEMORY' ), + ( True, ktReason_Unknown_File_Not_Found, + 'Error: failed to start machine. Error message: File not found. (VERR_FILE_NOT_FOUND)' ), + ( True, ktReason_Unknown_File_Not_Found, # lump it in with file-not-found for now. + 'Error: failed to start machine. Error message: Not supported. (VERR_NOT_SUPPORTED)' ), + ( False, ktReason_Unknown_VM_Crash, 'txsDoConnectViaTcp: Machine state: Aborted' ), + ( True, ktReason_Host_Modprobe_Failed, 'Kernel driver not installed' ), + ( True, ktReason_OSInstall_Sata_no_BM, 'PCHS=14128/14134/8224' ), + ( True, ktReason_Host_DoubleFreeHeap, 'double free or corruption' ), + #( False, ktReason_Unknown_VM_Start_Error, 'VMSetError: ' ), - false positives for stuff like: + # "VMSetError: VD: Backend 'VBoxIsoMaker' does not support async I/O" + ( False, ktReason_Unknown_VM_Start_Error, 'error: failed to open session for' ), + ( False, ktReason_Unknown_VM_Runtime_Error, 'Console: VM runtime error: fatal=true' ), + ]; + + ## This we search a main log for to figure out why something went bust. + katSimpleMainLogReasons = [ + # ( Whether to stop on hit, reason tuple, needle text. ) + ( False, ktReason_Host_win32com_gen_py, 'ModuleNotFoundError: No module named \'win32com.gen_py' ), + + ]; + + ## This we search a VM log for to figure out why something went bust. + katSimpleVmLogReasons = [ + # ( Whether to stop on hit, reason tuple, needle text. ) + # Note: Works for ATA and VD drivers. + ( False, ktReason_Host_DiskFull, '_DISKFULL' ), + ]; + + ## Things we search a VBoxHardening.log file for to figure out why something went bust. + katSimpleVBoxHardeningLogReasons = [ + # ( Whether to stop on hit, reason tuple, needle text. ) + ( True, ktReason_Host_DriverNotLoaded, 'Error opening VBoxDrvStub: STATUS_OBJECT_NAME_NOT_FOUND' ), + ( True, ktReason_Host_NotSignedWithBuildCert, 'Not signed with the build certificate' ), + ( True, ktReason_Host_TSTInfo_Accuracy_OOR, 'RTCRTSPTSTINFO::Accuracy::Millis: Out of range' ), + ( False, ktReason_Unknown_VM_Crash, 'Quitting: ExitCode=0xc0000005 (rcNtWait=' ), + ]; + + ## Things we search a kernel.log file for to figure out why something went bust. + katSimpleKernelLogReasons = [ + # ( Whether to stop on hit, reason tuple, needle text. ) + ( True, ktReason_Panic_HugeMemory, 'mm/huge_memory.c:1988' ), + ( True, ktReason_Panic_IOAPICDoesntWork, 'IO-APIC + timer doesn\'t work' ), + ( True, ktReason_Panic_TxUnitHang, 'Detected Tx Unit Hang' ), + ( True, ktReason_GuestBug_CompizVBoxQt, 'error 4 in libQt5CoreVBox' ), + ( True, ktReason_GuestBug_CompizVBoxQt, 'error 4 in libgtk-3' ), + ]; + + ## Things we search the _RIGHT_ _STRIPPED_ vgatext for. + katSimpleVgaTextReasons = [ + # ( Whether to stop on hit, reason tuple, needle text. ) + ( True, ktReason_Panic_MP_BIOS_IO_APIC, + "..MP-BIOS bug: 8254 timer not connected to IO-APIC\n\n" ), + ( True, ktReason_Panic_MP_BIOS_IO_APIC, + "..MP-BIOS bug: 8254 timer not connected to IO-APIC\n" + "...trying to set up timer (IRQ0) through the 8259A ... failed.\n" + "...trying to set up timer as Virtual Wire IRQ... failed.\n" + "...trying to set up timer as ExtINT IRQ... failed :(.\n" + "Kernel panic - not syncing: IO-APIC + timer doesn't work! Boot with apic=debug\n" + "and send a report. Then try booting with the 'noapic' option\n" + "\n" ), + ( True, ktReason_OSInstall_GRUB_hang, + "-----\nGRUB Loading stage2..\n\n\n\n" ), + ( True, ktReason_OSInstall_GRUB_hang, + "-----\nGRUB Loading stage2...\n\n\n\n" ), # the 3 dot hang appears to be less frequent + ( True, ktReason_OSInstall_GRUB_hang, + "-----\nGRUB Loading stage2....\n\n\n\n" ), # the 4 dot hang appears to be very infrequent + ( True, ktReason_OSInstall_GRUB_hang, + "-----\nGRUB Loading stage2.....\n\n\n\n" ), # the 5 dot hang appears to be more frequent again + ( True, ktReason_OSInstall_Udev_hang, + "\nStarting udev:\n\n\n\n" ), + ( True, ktReason_OSInstall_Udev_hang, + "\nStarting udev:\n------" ), + ( True, ktReason_Panic_BootManagerC000000F, + "Windows failed to start. A recent hardware or software change might be the" ), + ( True, ktReason_BootManager_Image_corrupt, + "BOOTMGR image is corrupt. The system cannot boot." ), + ]; + + ## Things we search for in the info.txt file. Require handlers for now. + katInfoTextHandlers = [ + # ( Trigger text, handler method ) + ( "kvm_lock_spinning", investigateInfoKvmLockSpinning ), + ( "HalReturnToFirmware", investigateInfoHalReturnToFirmware ), + ]; + + ## Mapping screenshot/failure SHA-256 hashes to failure reasons. + katSimpleScreenshotHashReasons = [ + # ( Whether to stop on hit, reason tuple, lowercased sha-256 of PIL.Image.tostring output ) + ( True, ktReason_BSOD_Recovery, '576f8e38d62b311cac7e3dc3436a0d0b9bd8cfd7fa9c43aafa95631520a45eac' ), + ( True, ktReason_BSOD_Automatic_Repair, 'c6a72076cc619937a7a39cfe9915b36d94cee0d4e3ce5ce061485792dcee2749' ), + ( True, ktReason_BSOD_Automatic_Repair, '26c4d8a724ff2c5e1051f3d5b650dbda7b5fdee0aa3e3c6059797f7484a515df' ), + ( True, ktReason_BSOD_0000007F, '57e1880619e13042a87100e7a38c8974b85ce3866501be621bea0cc696bb2c63' ), + ( True, ktReason_BSOD_000000D1, '134621281f00a3f8aeeb7660064bffbf6187ed56d5852142328d0bcb18ef0ede' ), + ( True, ktReason_BSOD_000000D1, '279f11258150c9d2fef041eca65501f3141da8df39256d8f6377e897e3b45a93' ), + ( True, ktReason_BSOD_C0000225, 'bd13a144be9dcdfb16bc863ff4c8f02a86e263c174f2cd5ffd27ca5f3aa31789' ), + ( True, ktReason_BSOD_C0000225, '8348b465e7ee9e59dd4e785880c57fd8677de05d11ac21e786bfde935307b42f' ), + ( True, ktReason_BSOD_C0000225, '1316e1fc818a73348412788e6910b8c016f237d8b4e15b20caf4a866f7a7840e' ), + ( True, ktReason_BSOD_C0000225, '54e0acbff365ce20a85abbe42bcd53647b8b9e80c68e45b2cd30e86bf177a0b5' ), + ( True, ktReason_BSOD_C0000225, '50fec50b5199923fa48b3f3e782687cc381e1c8a788ebda14e6a355fbe3bb1b3' ), + ]; + + + def scanLog(self, asLogs, atNeedles, oCaseFile, idTestResult): + """ + Scans for atNeedles in sLog. + + Returns True if a stop-on-hit neelde was found. + Returns None if a no-stop reason was found. + Returns False if no hit. + """ + fRet = False; + for fStopOnHit, tReason, oNeedle in atNeedles: + fMatch = False; + if utils.isString(oNeedle): + for sLog in asLogs: + if sLog: + fMatch |= sLog.find(oNeedle) > 0; + else: + for sLog in asLogs: + if sLog: + fMatch |= oNeedle.search(sLog) is not None; + if fMatch: + oCaseFile.noteReasonForId(tReason, idTestResult); + if fStopOnHit: + return True; + fRet = None; + return fRet; + + + def investigateGATest(self, oCaseFile, oFailedResult, sResultLog): + """ + Investigates a failed VM run. + """ + enmReason = None; + sParentName = oFailedResult.oParent.sName if oFailedResult.oParent else ''; + if oFailedResult.sName == 'VBoxWindowsAdditions.exe' or sResultLog.find('VBoxWindowsAdditions.exe" failed with') > 0: + enmReason = self.ktReason_Add_Installer_Win_Failed; + # guest control: + elif sParentName == 'Guest Control' and oFailedResult.sName == 'Preparations': + enmReason = self.ktReason_Add_GstCtl_Preparations; + elif oFailedResult.sName == 'Session Basics': + enmReason = self.ktReason_Add_GstCtl_SessionBasics; + elif oFailedResult.sName == 'Session Process References': + enmReason = self.ktReason_Add_GstCtl_SessionProcRefs; + elif oFailedResult.sName == 'Copy from guest': + if sResultLog.find('*** abort action ***') >= 0: + enmReason = self.ktReason_Add_GstCtl_CopyFromGuest_Timeout; + elif oFailedResult.sName == 'Copy to guest': + off = sResultLog.find('"Guest directory "'); + if off > 0 and sResultLog.find('" already exists"', off, off + 80): + enmReason = self.ktReason_Add_GstCtl_CopyToGuest_DstExists; + elif sResultLog.find('Guest destination must not be empty') >= 0: + enmReason = self.ktReason_Add_GstCtl_CopyToGuest_DstEmpty; + elif sResultLog.find('*** abort action ***') >= 0: + enmReason = self.ktReason_Add_GstCtl_CopyToGuest_Timeout; + elif oFailedResult.sName.find('Session w/ Guest Reboot') >= 0: + enmReason = self.ktReason_Add_GstCtl_Session_Reboot; + # shared folders: + elif sParentName == 'Shared Folders' and oFailedResult.sName == 'Automounting': + enmReason = self.ktReason_Add_ShFl_Automount; + elif oFailedResult.sName == 'mmap': + if sResultLog.find('FsPerf: Flush issue at offset ') >= 0: + enmReason = self.ktReason_Add_Mmap_Coherency; + elif sResultLog.find('FlushViewOfFile') >= 0: + enmReason = self.ktReason_Add_FlushViewOfFile; + elif sParentName == 'Shared Folders' and oFailedResult.sName == 'Running FsPerf': + enmReason = self.ktReason_Add_ShFl_FsPerf; ## Maybe it would be better to be more specific... + + if enmReason is not None: + return oCaseFile.noteReasonForId(enmReason, oFailedResult.idTestResult); + + self.vprint(u'TODO: Cannot place GA failure idTestResult=%u - %s' % (oFailedResult.idTestResult, oFailedResult.sName,)); + self.dprint(u'%s + %s <<\n%s\n<<' % (oFailedResult.tsCreated, oFailedResult.tsElapsed, sResultLog,)); + return False; + + def isResultFromGATest(self, oCaseFile, oFailedResult): + """ + Checks if this result and corresponding log snippet looks like a GA test run. + """ + while oFailedResult is not None: + if oFailedResult.sName in [ 'Guest Control', 'Shared Folders', 'FsPerf', 'VBoxWindowsAdditions.exe' ]: + return True; + if oCaseFile.oTestCase.sName == 'Guest Additions' and oFailedResult.sName in [ 'Install', ]: + return True; + oFailedResult = oFailedResult.oParent; + return False; + + + def investigateVMResult(self, oCaseFile, oFailedResult, sResultLog): + """ + Investigates a failed VM run. + """ + + def investigateLogSet(): + """ + Investigates the current set of VM related logs. + """ + self.dprint('investigateLogSet: log lengths: result %u, VM %u, kernel %u, vga text %u, info text %u, hard %u' + % ( len(sResultLog if sResultLog else ''), + len(sVMLog if sVMLog else ''), + len(sKrnlLog if sKrnlLog else ''), + len(sVgaText if sVgaText else ''), + len(sInfoText if sInfoText else ''), + len(sNtHardLog if sNtHardLog else ''),)); + + #self.dprint(u'main.log<<<\n%s\n<<<\n' % (sResultLog,)); + #self.dprint(u'vbox.log<<<\n%s\n<<<\n' % (sVMLog,)); + #self.dprint(u'krnl.log<<<\n%s\n<<<\n' % (sKrnlLog,)); + #self.dprint(u'vgatext.txt<<<\n%s\n<<<\n' % (sVgaText,)); + #self.dprint(u'info.txt<<<\n%s\n<<<\n' % (sInfoText,)); + #self.dprint(u'hard.txt<<<\n%s\n<<<\n' % (sNtHardLog,)); + + # TODO: more + + # + # Look for BSODs. Some stupid stupid inconsistencies in reason and log messages here, so don't try prettify this. + # + sDetails = self.findInAnyAndReturnRestOfLine([ sVMLog, sResultLog ], + 'GIM: HyperV: Guest indicates a fatal condition! P0='); + if sDetails is not None: + # P0=%#RX64 P1=%#RX64 P2=%#RX64 P3=%#RX64 P4=%#RX64 " + sKey = sDetails.split(' ', 1)[0]; + try: sKey = '0x%08X' % (int(sKey, 16),); + except: pass; + if sKey in self.asBsodReasons: + tReason = ( self.ksBsodCategory, sKey ); + elif sKey.lower() in self.asBsodReasons: # just in case. + tReason = ( self.ksBsodCategory, sKey.lower() ); + else: + self.dprint(u'BSOD "%s" not found in %s;' % (sKey, self.asBsodReasons)); + tReason = ( self.ksBsodCategory, self.ksBsodAddNew ); + return oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult, sComment = sDetails.strip()); + + fFoundSomething = False; + + # + # Look for linux panic. + # + if sKrnlLog is not None: + fRet = self.scanLog([sKrnlLog,], self.katSimpleKernelLogReasons, oCaseFile, oFailedResult.idTestResult); + if fRet is True: + return fRet; + fFoundSomething |= fRet is None; + + # + # Loop thru the simple stuff. + # + + # Main log. + fRet = self.scanLog([sResultLog,], self.katSimpleMainLogReasons, oCaseFile, oFailedResult.idTestResult); + if fRet is True: + return fRet; + fFoundSomething |= fRet is None; + + # VM log. + fRet = self.scanLog([sVMLog,], self.katSimpleVmLogReasons, oCaseFile, oFailedResult.idTestResult); + if fRet is True: + return fRet; + fFoundSomething |= fRet is None; + + # Old main + vm log. + fRet = self.scanLog([sResultLog, sVMLog], self.katSimpleMainAndVmLogReasonsDeprecated, + oCaseFile, oFailedResult.idTestResult); + if fRet is True: + return fRet; + fFoundSomething |= fRet is None; + + # Continue with vga text. + if sVgaText: + fRet = self.scanLog([sVgaText,], self.katSimpleVgaTextReasons, oCaseFile, oFailedResult.idTestResult); + if fRet is True: + return fRet; + fFoundSomething |= fRet is None; + + # Continue with screen hashes. + if sScreenHash is not None: + for fStopOnHit, tReason, sHash in self.katSimpleScreenshotHashReasons: + if sScreenHash == sHash: + oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult); + if fStopOnHit: + return True; + fFoundSomething = True; + + # Check VBoxHardening.log. + if sNtHardLog is not None: + fRet = self.scanLog([sNtHardLog,], self.katSimpleVBoxHardeningLogReasons, oCaseFile, oFailedResult.idTestResult); + if fRet is True: + return fRet; + fFoundSomething |= fRet is None; + + # + # Complicated stuff. + # + dLogs = { + 'sVMLog': sVMLog, + 'sNtHardLog': sNtHardLog, + 'sScreenHash': sScreenHash, + 'sKrnlLog': sKrnlLog, + 'sVgaText': sVgaText, + 'sInfoText': sInfoText, + }; + + # info.txt. + if sInfoText: + for sNeedle, fnHandler in self.katInfoTextHandlers: + if sInfoText.find(sNeedle) > 0: + (fStop, tReason) = fnHandler(self, oCaseFile, sInfoText, dLogs); + if tReason is not None: + oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult); + if fStop: + return True; + fFoundSomething = True; + + # + # Check for repeated reboots... + # + if sVMLog is not None: + cResets = sVMLog.count('Changing the VM state from \'RUNNING\' to \'RESETTING\''); + if cResets > 10: + return oCaseFile.noteReasonForId(self.ktReason_Unknown_Reboot_Loop, oFailedResult.idTestResult, + sComment = 'Counted %s reboots' % (cResets,)); + + return fFoundSomething; + + # + # Check if we got any VM or/and kernel logs. Treat them as sets in + # case we run multiple VMs here (this is of course ASSUMING they + # appear in the order that terminateVmBySession uploads them). + # + cTimes = 0; + sVMLog = None; + sNtHardLog = None; + sScreenHash = None; + sKrnlLog = None; + sVgaText = None; + sInfoText = None; + for oFile in oFailedResult.aoFiles: + if oFile.sKind == TestResultFileData.ksKind_LogReleaseVm: + if 'VBoxHardening.log' not in oFile.sFile: + if sVMLog is not None: + if investigateLogSet() is True: + return True; + cTimes += 1; + sInfoText = None; + sVgaText = None; + sKrnlLog = None; + sScreenHash = None; + sNtHardLog = None; + sVMLog = oCaseFile.getLogFile(oFile); + else: + sNtHardLog = oCaseFile.getLogFile(oFile); + elif oFile.sKind == TestResultFileData.ksKind_LogGuestKernel: + sKrnlLog = oCaseFile.getLogFile(oFile); + elif oFile.sKind == TestResultFileData.ksKind_InfoVgaText: + sVgaText = '\n'.join([sLine.rstrip() for sLine in oCaseFile.getLogFile(oFile).split('\n')]); + elif oFile.sKind == TestResultFileData.ksKind_InfoCollection: + sInfoText = oCaseFile.getLogFile(oFile); + elif oFile.sKind == TestResultFileData.ksKind_ScreenshotFailure: + sScreenHash = oCaseFile.getScreenshotSha256(oFile); + if sScreenHash is not None: + sScreenHash = sScreenHash.lower(); + self.vprint(u'%s %s' % ( sScreenHash, oFile.sFile,)); + + if ( sVMLog is not None \ + or sNtHardLog is not None \ + or cTimes == 0) \ + and investigateLogSet() is True: + return True; + + return None; + + def isResultFromVMRun(self, oFailedResult, sResultLog): + """ + Checks if this result and corresponding log snippet looks like a VM run. + """ + + # Look for startVmEx/ startVmAndConnectToTxsViaTcp and similar output in the log. + if sResultLog.find(' startVm') > 0: + return True; + + # Any other indicators? No? + _ = oFailedResult; + return False; + + + ## Things we search a VBoxSVC log for to figure out why something went bust. + katSimpleSvcLogReasons = [ + # ( Whether to stop on hit, reason tuple, needle text. ) + ( False, ktReason_Unknown_VM_Crash, re.compile(r'Reaper.* exited normally: -1073741819 \(0xc0000005\)') ), + ( False, ktReason_Unknown_VM_Crash, re.compile(r'Reaper.* was signalled: 11 \(0xb\)') ), # For VBox < 6.1. + ( False, ktReason_Unknown_VM_Crash, re.compile(r'Reaper.* was signalled: SIGABRT.*') ), # Since VBox 7.0. + ( False, ktReason_Unknown_VM_Crash, re.compile(r'Reaper.* was signalled: SIGSEGV.*') ), + ( False, ktReason_Unknown_VM_Terminated, re.compile(r'Reaper.* was signalled: SIGTERM.*') ), + ( False, ktReason_Unknown_VM_Terminated, re.compile(r'Reaper.* was signalled: SIGKILL.*') ), + ]; + + def investigateSvcLogForVMRun(self, oCaseFile, sSvcLog): + """ + Check the VBoxSVC log for a single VM run. + """ + if sSvcLog: + fRet = self.scanLog([sSvcLog,], self.katSimpleSvcLogReasons, oCaseFile, oCaseFile.oTree.idTestResult); + if fRet is True or fRet is None: + return True; + return False; + + def investigateNtHardLogForVMRun(self, oCaseFile): + """ + Check if the hardening log for a single VM run contains VM crash indications. + """ + aoLogFiles = oCaseFile.oTree.getListOfLogFilesByKind(TestResultFileData.ksKind_LogReleaseVm); + for oLogFile in aoLogFiles: + if oLogFile.sFile.find('VBoxHardening.log') >= 0: + sLog = oCaseFile.getLogFile(oLogFile); + if sLog.find('Quitting: ExitCode=0xc0000005') >= 0: + return oCaseFile.noteReasonForId(self.ktReason_Unknown_VM_Crash, oCaseFile.oTree.idTestResult); + return False; + + + def investigateVBoxVMTest(self, oCaseFile, fSingleVM): + """ + Checks out a VBox VM test. + + This is generic investigation of a test running one or more VMs, like + for example a smoke test or a guest installation test. + + The fSingleVM parameter is a hint, which probably won't come in useful. + """ + _ = fSingleVM; + + # + # Get a list of test result failures we should be looking into and the main log. + # + aoFailedResults = oCaseFile.oTree.getListOfFailures(); + sMainLog = oCaseFile.getMainLog(); + + # + # There are a set of errors ending up on the top level result record. + # Should deal with these first. + # + if len(aoFailedResults) == 1 and aoFailedResults[0] == oCaseFile.oTree: + # Check if we've just got that XPCOM client smoke test shutdown issue. This will currently always + # be reported on the top result because vboxinstall.py doesn't add an error for it. It is easy to + # ignore other failures in the test if we're not a little bit careful here. + if sMainLog.find('vboxinstaller: Exit code: -11 (') > 0: + oCaseFile.noteReason(self.ktReason_XPCOM_Exit_Minus_11); + return self.caseClosed(oCaseFile); + + # Hang after starting VBoxSVC (e.g. idTestSet=136307258) + if self.isThisFollowedByTheseLines(sMainLog, 'oVBoxMgr= 0: + oCaseFile.noteReason(self.ktReason_XPCOM_VBoxSVC_Hang_Plus_Heap_Corruption); + else: + oCaseFile.noteReason(self.ktReason_XPCOM_VBoxSVC_Hang); + return self.caseClosed(oCaseFile); + + # Look for heap corruption without visible hang. + if sMainLog.find('*** glibc detected *** /') > 0 \ + or sMainLog.find("-1073740940") > 0: # STATUS_HEAP_CORRUPTION / 0xc0000374 + oCaseFile.noteReason(self.ktReason_Unknown_Heap_Corruption); + return self.caseClosed(oCaseFile); + + # Out of memory w/ timeout. + if sMainLog.find('sErrId=HostMemoryLow') > 0: + oCaseFile.noteReason(self.ktReason_Host_HostMemoryLow); + return self.caseClosed(oCaseFile); + + # Stale files like vts_rm.exe (windows). + offEnd = sMainLog.rfind('*** The test driver exits successfully. ***'); + if offEnd > 0 and sMainLog.find('[Error 145] The directory is not empty: ', offEnd) > 0: + oCaseFile.noteReason(self.ktReason_Ignore_Stale_Files); + return self.caseClosed(oCaseFile); + + # + # XPCOM screwup + # + if sMainLog.find('AttributeError: \'NoneType\' object has no attribute \'addObserver\'') > 0: + oCaseFile.noteReason(self.ktReason_Buggy_Build_Broken_Build); + return self.caseClosed(oCaseFile); + + # + # Go thru each failed result. + # + for oFailedResult in aoFailedResults: + self.dprint(u'Looking at test result #%u - %s' % (oFailedResult.idTestResult, oFailedResult.getFullName(),)); + sResultLog = TestSetData.extractLogSectionElapsed(sMainLog, oFailedResult.tsCreated, oFailedResult.tsElapsed); + if oFailedResult.sName == 'Installing VirtualBox': + self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = True) + + elif oFailedResult.sName == 'Uninstalling VirtualBox': + self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = False) + + elif self.isResultFromVMRun(oFailedResult, sResultLog): + self.investigateVMResult(oCaseFile, oFailedResult, sResultLog); + + elif self.isResultFromGATest(oCaseFile, oFailedResult): + self.investigateGATest(oCaseFile, oFailedResult, sResultLog); + + elif sResultLog.find('most likely not unique') > 0: + oCaseFile.noteReasonForId(self.ktReason_Host_NetworkMisconfiguration, oFailedResult.idTestResult) + elif sResultLog.find('Exception: 0x800706be (Call to remote object failed (NS_ERROR_CALL_FAILED))') > 0: + oCaseFile.noteReasonForId(self.ktReason_XPCOM_NS_ERROR_CALL_FAILED, oFailedResult.idTestResult); + + elif sResultLog.find('The machine is not mutable (state is ') > 0: + self.vprint('Ignoring "machine not mutable" error as it is probably due to an earlier problem'); + oCaseFile.noteReasonForId(self.ktHarmless, oFailedResult.idTestResult); + + elif sResultLog.find('** error: no action was specified') > 0 \ + or sResultLog.find('(len(self._asXml, asText))') > 0: + oCaseFile.noteReasonForId(self.ktReason_Ignore_Buggy_Test_Driver, oFailedResult.idTestResult); + + else: + self.vprint(u'TODO: Cannot place idTestResult=%u - %s' % (oFailedResult.idTestResult, oFailedResult.sName,)); + self.dprint(u'%s + %s <<\n%s\n<<' % (oFailedResult.tsCreated, oFailedResult.tsElapsed, sResultLog,)); + + # + # Windows python/com screwup. + # + if sMainLog.find('ModuleNotFoundError: No module named \'win32com.gen_py') > 0: + oCaseFile.noteReason(self.ktReason_Host_win32com_gen_py); + return self.caseClosed(oCaseFile); + + # + # Check VBoxSVC.log and VBoxHardening.log for VM crashes if inconclusive on single VM runs. + # + if fSingleVM and len(oCaseFile.dReasonForResultId) < len(aoFailedResults): + self.dprint(u'Got %u out of %u - checking VBoxSVC.log...' + % (len(oCaseFile.dReasonForResultId), len(aoFailedResults))); + if self.investigateSvcLogForVMRun(oCaseFile, oCaseFile.getSvcLog()): + return self.caseClosed(oCaseFile); + if self.investigateNtHardLogForVMRun(oCaseFile): + return self.caseClosed(oCaseFile); + + # + # Report home and close the case if we got them all, otherwise log it. + # + if len(oCaseFile.dReasonForResultId) >= len(aoFailedResults): + return self.caseClosed(oCaseFile); + + if oCaseFile.dReasonForResultId: + self.vprint(u'TODO: Got %u out of %u - close, but no cigar. :-/' + % (len(oCaseFile.dReasonForResultId), len(aoFailedResults))); + else: + self.vprint(u'XXX: Could not figure out anything at all! :-('); + return False; + + + ## Things we search a main log for to figure out why something in the API test went bust. + katSimpleApiMainLogReasons = [ + # ( Whether to stop on hit, reason tuple, needle text. ) + ( True, ktReason_Networking_Nonexistent_host_nic, + 'rc=E_FAIL text="Nonexistent host networking interface, name \'eth0\' (VERR_INTERNAL_ERROR)"' ), + ( False, ktReason_XPCOM_NS_ERROR_CALL_FAILED, + 'Exception: 0x800706be (Call to remote object failed (NS_ERROR_CALL_FAILED))' ), + ( True, ktReason_API_std_bad_alloc, 'Unexpected exception: std::bad_alloc' ), + ( True, ktReason_API_Digest_Mismatch, 'Digest mismatch (VERR_NOT_EQUAL)' ), + ( True, ktReason_API_MoveVM_SharingViolation, 'rc=VBOX_E_IPRT_ERROR text="Could not copy the log file ' ), + ( True, ktReason_API_MoveVM_InvalidParameter, + 'rc=VBOX_E_IPRT_ERROR text="Could not copy the setting file ' ), + ( True, ktReason_API_Open_Session_Failed, 'error: failed to open session for' ), + ]; + + def investigateVBoxApiTest(self, oCaseFile): + """ + Checks out a VBox API test. + """ + + # + # Get a list of test result failures we should be looking into and the main log. + # + aoFailedResults = oCaseFile.oTree.getListOfFailures(); + sMainLog = oCaseFile.getMainLog(); + + # + # Go thru each failed result. + # + for oFailedResult in aoFailedResults: + self.dprint(u'Looking at test result #%u - %s' % (oFailedResult.idTestResult, oFailedResult.getFullName(),)); + sResultLog = TestSetData.extractLogSectionElapsed(sMainLog, oFailedResult.tsCreated, oFailedResult.tsElapsed); + if oFailedResult.sName == 'Installing VirtualBox': + self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = True) + + elif oFailedResult.sName == 'Uninstalling VirtualBox': + self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = False) + + elif sResultLog.find('Exception: 0x800706be (Call to remote object failed (NS_ERROR_CALL_FAILED))') > 0: + oCaseFile.noteReasonForId(self.ktReason_XPCOM_NS_ERROR_CALL_FAILED, oFailedResult.idTestResult); + + else: + fFoundSomething = False; + for fStopOnHit, tReason, sNeedle in self.katSimpleApiMainLogReasons: + if sResultLog.find(sNeedle) > 0: + oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult); + fFoundSomething = True; + if fStopOnHit: + break; + if fFoundSomething: + self.vprint(u'TODO: Cannot place idTestResult=%u - %s' % (oFailedResult.idTestResult, oFailedResult.sName,)); + self.dprint(u'%s + %s <<\n%s\n<<' % (oFailedResult.tsCreated, oFailedResult.tsElapsed, sResultLog,)); + + # + # Report home and close the case if we got them all, otherwise log it. + # + if len(oCaseFile.dReasonForResultId) >= len(aoFailedResults): + return self.caseClosed(oCaseFile); + + if oCaseFile.dReasonForResultId: + self.vprint(u'TODO: Got %u out of %u - close, but no cigar. :-/' + % (len(oCaseFile.dReasonForResultId), len(aoFailedResults))); + else: + self.vprint(u'XXX: Could not figure out anything at all! :-('); + return False; + + + def reasoningFailures(self): + """ + Guess the reason for failures. + """ + # + # Get a list of failed test sets without any assigned failure reason. + # + cGot = 0; + if not self.oConfig.aidTestSets: + aoTestSets = self.oTestSetLogic.fetchFailedSetsWithoutReason(cHoursBack = self.oConfig.cHoursBack, + tsNow = self.tsNow); + else: + aoTestSets = [self.oTestSetLogic.getById(idTestSet) for idTestSet in self.oConfig.aidTestSets]; + for oTestSet in aoTestSets: + self.dprint(u'----------------------------------- #%u, status %s -----------------------------------' + % ( oTestSet.idTestSet, oTestSet.enmStatus,)); + + # + # Open a case file and assign it to the right investigator. + # + (oTree, _ ) = self.oTestResultLogic.fetchResultTree(oTestSet.idTestSet); + oBuild = BuildDataEx().initFromDbWithId( self.oDb, oTestSet.idBuild, oTestSet.tsCreated); + oTestBox = TestBoxData().initFromDbWithGenId( self.oDb, oTestSet.idGenTestBox); + oTestGroup = TestGroupData().initFromDbWithId( self.oDb, oTestSet.idTestGroup, oTestSet.tsCreated); + oTestCase = TestCaseDataEx().initFromDbWithGenId( self.oDb, oTestSet.idGenTestCase, oTestSet.tsConfig); + + oCaseFile = VirtualTestSheriffCaseFile(self, oTestSet, oTree, oBuild, oTestBox, oTestGroup, oTestCase); + + if oTestSet.enmStatus == TestSetData.ksTestStatus_BadTestBox: + self.dprint(u'investigateBadTestBox is taking over %s.' % (oCaseFile.sLongName,)); + fRc = self.investigateBadTestBox(oCaseFile); + + elif oCaseFile.isVBoxUnitTest(): + self.dprint(u'investigateVBoxUnitTest is taking over %s.' % (oCaseFile.sLongName,)); + fRc = self.investigateVBoxUnitTest(oCaseFile); + + elif oCaseFile.isVBoxInstallTest() or oCaseFile.isVBoxUnattendedInstallTest(): + self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,)); + fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True); + + elif oCaseFile.isVBoxUSBTest(): + self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,)); + fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True); + + elif oCaseFile.isVBoxStorageTest(): + self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,)); + fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True); + + elif oCaseFile.isVBoxGAsTest(): + self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,)); + fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True); + + elif oCaseFile.isVBoxAPITest(): + self.dprint(u'investigateVBoxApiTest is taking over %s.' % (oCaseFile.sLongName,)); + fRc = self.investigateVBoxApiTest(oCaseFile); + + elif oCaseFile.isVBoxBenchmarkTest(): + self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,)); + fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = False); + + elif oCaseFile.isVBoxSmokeTest(): + self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,)); + fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = False); + + elif oCaseFile.isVBoxSerialTest(): + self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,)); + fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = False); + + else: + self.vprint(u'reasoningFailures: Unable to classify test set: %s' % (oCaseFile.sLongName,)); + fRc = False; + cGot += fRc is True; + + self.vprint(u'reasoningFailures: Got %u out of %u' % (cGot, len(aoTestSets), )); + return 0; + + + def main(self): + """ + The 'main' function. + Return exit code (0, 1, etc). + """ + # Database stuff. + self.oDb = TMDatabaseConnection() + self.oTestResultLogic = TestResultLogic(self.oDb); + self.oTestSetLogic = TestSetLogic(self.oDb); + self.oFailureReasonLogic = FailureReasonLogic(self.oDb); + self.oTestResultFailureLogic = TestResultFailureLogic(self.oDb); + self.asBsodReasons = self.oFailureReasonLogic.fetchForSheriffByNamedCategory(self.ksBsodCategory); + self.asUnitTestReasons = self.oFailureReasonLogic.fetchForSheriffByNamedCategory(self.ksUnitTestCategory); + + # Get a fix on our 'now' before we do anything.. + self.oDb.execute('SELECT CURRENT_TIMESTAMP - interval \'%s hours\'', (self.oConfig.cStartHoursAgo,)); + self.tsNow = self.oDb.fetchOne(); + + # If we're suppost to commit anything we need to get our user ID. + rcExit = 0; + if self.oConfig.fRealRun: + self.oLogin = UserAccountLogic(self.oDb).tryFetchAccountByLoginName(VirtualTestSheriff.ksLoginName); + if self.oLogin is None: + rcExit = self.eprint('Cannot find my user account "%s"!' % (VirtualTestSheriff.ksLoginName,)); + else: + self.uidSelf = self.oLogin.uid; + + # + # Do the stuff. + # + if rcExit == 0: + rcExit = self.selfCheck(); + if rcExit == 0: + rcExit = self.badTestBoxManagement(); + rcExit2 = self.reasoningFailures(); + if rcExit == 0: + rcExit = rcExit2; + # Redo the bad testbox management after failure reasons have been assigned (got timing issues). + if rcExit == 0: + rcExit = self.badTestBoxManagement(); + + # Cleanup. + self.oFailureReasonLogic = None; + self.oTestResultFailureLogic = None; + self.oTestSetLogic = None; + self.oTestResultLogic = None; + self.oDb.close(); + self.oDb = None; + if self.oLogFile is not None: + self.oLogFile.close(); + self.oLogFile = None; + return rcExit; + +if __name__ == '__main__': + sys.exit(VirtualTestSheriff().main()); -- cgit v1.2.3