Adding upstream version 6.0.4-dfsg.upstream/6.0.4-dfsgupstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

9 files changed, 2473 insertions, 0 deletions

diff --git a/src/VBox/ValidationKit/testmanager/batch/Makefile.kmk b/src/VBox/ValidationKit/testmanager/batch/Makefile.kmk
new file mode 100644
index 00000000..6f03da64
--- /dev/null
+++ b/src/VBox/ValidationKit/testmanager/batch/Makefile.kmk
@@ -0,0 +1,36 @@
+# $Id: Makefile.kmk $
+## @file
+# VirtualBox Validation Kit.
+#
+
+#
+# Copyright (C) 2006-2019 Oracle Corporation
+#
+# This file is part of VirtualBox Open Source Edition (OSE), as
+# available from http://www.virtualbox.org. This file is free software;
+# you can redistribute it and/or modify it under the terms of the GNU
+# General Public License (GPL) as published by the Free Software
+# Foundation, in version 2 as it comes in the "COPYING" file of the
+# VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+# hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+#
+# The contents of this file may alternatively be used under the terms
+# of the Common Development and Distribution License Version 1.0
+# (CDDL) only, as it comes in the "COPYING.CDDL" file of the
+# VirtualBox OSE distribution, in which case the provisions of the
+# CDDL are applicable instead of those of the GPL.
+#
+# You may elect to license modified versions of this file under the
+# terms and conditions of either the GPL or the CDDL or both.
+#
+
+SUB_DEPTH = ../../../../..
+include $(KBUILD_PATH)/subheader.kmk
+
+
+VBOX_VALIDATIONKIT_PYTHON_SOURCES += $(wildcard $(PATH_SUB_CURRENT)/*.py)
+
+$(evalcall def_vbox_validationkit_process_python_sources)
+$(evalcall def_vbox_validationkit_process_js_sources)
+include $(FILE_KBUILD_SUB_FOOTER)
+
diff --git a/src/VBox/ValidationKit/testmanager/batch/add_build.py b/src/VBox/ValidationKit/testmanager/batch/add_build.py
new file mode 100755
index 00000000..0cdb8b45
--- /dev/null
+++ b/src/VBox/ValidationKit/testmanager/batch/add_build.py
@@ -0,0 +1,127 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# $Id: add_build.py $
+# pylint: disable=C0301
+
+"""
+Interface used by the tinderbox server side software to add a fresh build.
+"""
+
+__copyright__ = \
+"""
+Copyright (C) 2012-2019 Oracle Corporation
+
+This file is part of VirtualBox Open Source Edition (OSE), as
+available from http://www.virtualbox.org. This file is free software;
+you can redistribute it and/or modify it under the terms of the GNU
+General Public License (GPL) as published by the Free Software
+Foundation, in version 2 as it comes in the "COPYING" file of the
+VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+
+The contents of this file may alternatively be used under the terms
+of the Common Development and Distribution License Version 1.0
+(CDDL) only, as it comes in the "COPYING.CDDL" file of the
+VirtualBox OSE distribution, in which case the provisions of the
+CDDL are applicable instead of those of the GPL.
+
+You may elect to license modified versions of this file under the
+terms and conditions of either the GPL or the CDDL or both.
+"""
+__version__ = "$Revision: 127855 $"
+
+# Standard python imports
+import sys;
+import os;
+from optparse import OptionParser;  # pylint: disable=deprecated-module
+
+# Add Test Manager's modules path
+g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))));
+sys.path.append(g_ksTestManagerDir);
+
+# Test Manager imports
+from testmanager.core.db    import TMDatabaseConnection;
+from testmanager.core.build import BuildDataEx, BuildLogic, BuildCategoryData;
+
+class Build(object): # pylint: disable=R0903
+    """
+    Add build info into Test Manager database.
+    """
+
+    def __init__(self):
+        """
+        Parse command line.
+        """
+
+        oParser = OptionParser();
+        oParser.add_option('-q', '--quiet', dest = 'fQuiet', action = 'store_true',
+                           help = 'Quiet execution');
+        oParser.add_option('-b', '--branch', dest = 'sBranch', metavar = '<branch>',
+                           help = 'branch name (default: trunk)', default = 'trunk');
+        oParser.add_option('-p', '--product', dest = 'sProductName', metavar = '<name>',
+                           help = 'The product name.');
+        oParser.add_option('-r', '--revision', dest = 'iRevision', metavar = '<rev>',
+                           help = 'revision number');
+        oParser.add_option('-R', '--repository', dest = 'sRepository', metavar = '<repository>',
+                           help = 'Version control repository name.');
+        oParser.add_option('-t', '--type', dest = 'sBuildType', metavar = '<type>',
+                           help = 'build type (debug, release etc.)');
+        oParser.add_option('-v', '--version', dest = 'sProductVersion', metavar = '<ver>',
+                           help = 'The product version number (suitable for RTStrVersionCompare)');
+        oParser.add_option('-o', '--os-arch', dest = 'asTargetOsArches', metavar = '<os.arch>', action = 'append',
+                           help = 'Target OS and architecture. This option can be repeated.');
+        oParser.add_option('-l', '--log', dest = 'sBuildLogPath', metavar = '<url>',
+                           help = 'URL to the build logs (optional).');
+        oParser.add_option('-f', '--file', dest = 'asFiles', metavar = '<file|url>', action = 'append',
+                           help = 'URLs or build share relative path to a build output file. This option can be repeated.');
+
+        (self.oConfig, _) = oParser.parse_args();
+
+        # Check command line
+        asMissing = [];
+        if self.oConfig.sBranch is None:            asMissing.append('--branch');
+        if self.oConfig.iRevision is None:          asMissing.append('--revision');
+        if self.oConfig.sProductVersion is None:    asMissing.append('--version');
+        if self.oConfig.sProductName is None:       asMissing.append('--product');
+        if self.oConfig.sBuildType is None:         asMissing.append('--type');
+        if self.oConfig.asTargetOsArches is None:   asMissing.append('--os-arch');
+        if self.oConfig.asFiles is None:            asMissing.append('--file');
+        if asMissing:
+            sys.stderr.write('syntax error: Missing: %s\n' % (asMissing,));
+            sys.exit(1);
+        # Temporary default.
+        if self.oConfig.sRepository is None:
+            self.oConfig.sRepository = 'vbox';
+
+    def add(self):
+        """
+        Add build data record into database.
+        """
+        oDb = TMDatabaseConnection()
+
+        # Assemble the build data.
+        oBuildData = BuildDataEx()
+        oBuildData.idBuildCategory    = None;
+        oBuildData.iRevision          = self.oConfig.iRevision
+        oBuildData.sVersion           = self.oConfig.sProductVersion
+        oBuildData.sLogUrl            = self.oConfig.sBuildLogPath
+        oBuildData.sBinaries          = ','.join(self.oConfig.asFiles);
+        oBuildData.oCat = BuildCategoryData().initFromValues(sProduct    = self.oConfig.sProductName,
+                                                             sRepository = self.oConfig.sRepository,
+                                                             sBranch     = self.oConfig.sBranch,
+                                                             sType       = self.oConfig.sBuildType,
+                                                             asOsArches  = self.oConfig.asTargetOsArches);
+
+        # Add record to database
+        try:
+            BuildLogic(oDb).addEntry(oBuildData, fCommit = True);
+        except:
+            if self.oConfig.fQuiet:
+                sys.exit(1);
+            raise;
+        oDb.close();
+        return 0;
+
+if __name__ == '__main__':
+    sys.exit(Build().add());
+
diff --git a/src/VBox/ValidationKit/testmanager/batch/check_for_deleted_builds.py b/src/VBox/ValidationKit/testmanager/batch/check_for_deleted_builds.py
new file mode 100755
index 00000000..8d12ff20
--- /dev/null
+++ b/src/VBox/ValidationKit/testmanager/batch/check_for_deleted_builds.py
@@ -0,0 +1,123 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# $Id: check_for_deleted_builds.py $
+# pylint: disable=C0301
+
+"""
+Admin job for checking detecting deleted builds.
+
+This is necessary when the tinderbox <-> test manager interface was
+busted and the build info in is out of sync.  The result is generally
+a lot of skipped tests because of missing builds, typically during
+bisecting problems.
+"""
+
+from __future__ import print_function;
+
+__copyright__ = \
+"""
+Copyright (C) 2012-2019 Oracle Corporation
+
+This file is part of VirtualBox Open Source Edition (OSE), as
+available from http://www.virtualbox.org. This file is free software;
+you can redistribute it and/or modify it under the terms of the GNU
+General Public License (GPL) as published by the Free Software
+Foundation, in version 2 as it comes in the "COPYING" file of the
+VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+
+The contents of this file may alternatively be used under the terms
+of the Common Development and Distribution License Version 1.0
+(CDDL) only, as it comes in the "COPYING.CDDL" file of the
+VirtualBox OSE distribution, in which case the provisions of the
+CDDL are applicable instead of those of the GPL.
+
+You may elect to license modified versions of this file under the
+terms and conditions of either the GPL or the CDDL or both.
+"""
+__version__ = "$Revision: 127855 $"
+
+# Standard python imports
+import sys;
+import os;
+from optparse import OptionParser;  # pylint: disable=deprecated-module
+
+# Add Test Manager's modules path
+g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))));
+sys.path.append(g_ksTestManagerDir);
+
+# Test Manager imports
+from testmanager.core.db    import TMDatabaseConnection;
+from testmanager.core.build import BuildLogic;
+
+
+
+class BuildChecker(object): # pylint: disable=R0903
+    """
+    Add build info into Test Manager database.
+    """
+
+    def __init__(self):
+        """
+        Parse command line.
+        """
+
+        oParser = OptionParser();
+        oParser.add_option('-q', '--quiet', dest = 'fQuiet', action = 'store_true',  default = False,
+                           help = 'Quiet execution');
+        oParser.add_option('--dry-run', dest = 'fRealRun',   action = 'store_false', default = False,
+                           help = 'Dry run');
+        oParser.add_option('--real-run', dest = 'fRealRun',  action = 'store_true',  default = False,
+                           help = 'Real run');
+
+        (self.oConfig, _) = oParser.parse_args();
+        if not self.oConfig.fQuiet:
+            if not self.oConfig.fRealRun:
+                print('Dry run.');
+            else:
+                print('Real run! Will commit findings!');
+
+
+    def checkBuilds(self):
+        """
+        Add build data record into database.
+        """
+        oDb = TMDatabaseConnection();
+        oBuildLogic = BuildLogic(oDb);
+
+        tsNow    = oDb.getCurrentTimestamp();
+        cMaxRows = 1024;
+        iStart   = 0;
+        while True:
+            aoBuilds = oBuildLogic.fetchForListing(iStart, cMaxRows, tsNow);
+            if not self.oConfig.fQuiet and aoBuilds:
+                print('Processing builds #%s thru #%s' % (aoBuilds[0].idBuild, aoBuilds[-1].idBuild));
+
+            for oBuild in aoBuilds:
+                if oBuild.fBinariesDeleted is False:
+                    rc = oBuild.areFilesStillThere();
+                    if rc is False:
+                        if not self.oConfig.fQuiet:
+                            print('missing files for build #%s / r%s / %s / %s / %s / %s / %s'
+                                  % (oBuild.idBuild, oBuild.iRevision, oBuild.sVersion, oBuild.oCat.sType,
+                                     oBuild.oCat.sBranch, oBuild.oCat.sProduct, oBuild.oCat.asOsArches,));
+                            print('  %s' % (oBuild.sBinaries,));
+                        if self.oConfig.fRealRun is True:
+                            oBuild.fBinariesDeleted = True;
+                            oBuildLogic.editEntry(oBuild, fCommit = True);
+                    elif rc is True and not self.oConfig.fQuiet:
+                        print('build #%s still have its files' % (oBuild.idBuild,));
+                    elif rc is None and not self.oConfig.fQuiet:
+                        print('Unable to determine state of build #%s' % (oBuild.idBuild,));
+
+            # advance
+            if len(aoBuilds) < cMaxRows:
+                break;
+            iStart += len(aoBuilds);
+
+        oDb.close();
+        return 0;
+
+if __name__ == '__main__':
+    sys.exit(BuildChecker().checkBuilds());
+
diff --git a/src/VBox/ValidationKit/testmanager/batch/close_orphaned_testsets.py b/src/VBox/ValidationKit/testmanager/batch/close_orphaned_testsets.py
new file mode 100755
index 00000000..331e3125
--- /dev/null
+++ b/src/VBox/ValidationKit/testmanager/batch/close_orphaned_testsets.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# $Id: close_orphaned_testsets.py $
+# pylint: disable=C0301
+
+"""
+Maintenance tool for closing orphaned testsets.
+"""
+
+from __future__ import print_function;
+
+__copyright__ = \
+"""
+Copyright (C) 2012-2019 Oracle Corporation
+
+This file is part of VirtualBox Open Source Edition (OSE), as
+available from http://www.virtualbox.org. This file is free software;
+you can redistribute it and/or modify it under the terms of the GNU
+General Public License (GPL) as published by the Free Software
+Foundation, in version 2 as it comes in the "COPYING" file of the
+VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+
+The contents of this file may alternatively be used under the terms
+of the Common Development and Distribution License Version 1.0
+(CDDL) only, as it comes in the "COPYING.CDDL" file of the
+VirtualBox OSE distribution, in which case the provisions of the
+CDDL are applicable instead of those of the GPL.
+
+You may elect to license modified versions of this file under the
+terms and conditions of either the GPL or the CDDL or both.
+"""
+__version__ = "$Revision: 127855 $"
+
+# Standard python imports
+import sys
+import os
+from optparse import OptionParser;  # pylint: disable=deprecated-module
+
+# Add Test Manager's modules path
+g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.append(g_ksTestManagerDir)
+
+# Test Manager imports
+from testmanager.core.db        import TMDatabaseConnection
+from testmanager.core.testset   import TestSetLogic;
+
+
+class CloseOrphanedTestSets(object):
+    """
+    Finds and closes orphaned testsets.
+    """
+
+    def __init__(self):
+        """
+        Parse command line
+        """
+        oParser = OptionParser();
+        oParser.add_option('-d', '--just-do-it', dest='fJustDoIt', action='store_true',
+                           help='Do the database changes.');
+
+
+        (self.oConfig, _) = oParser.parse_args();
+
+
+    def main(self):
+        """ Main method. """
+        oDb = TMDatabaseConnection();
+
+        # Get a list of orphans.
+        oLogic = TestSetLogic(oDb);
+        aoOrphans = oLogic.fetchOrphaned();
+        if aoOrphans:
+            # Complete them.
+            if self.oConfig.fJustDoIt:
+                print('Completing %u test sets as abandoned:' % (len(aoOrphans),));
+                for oTestSet in aoOrphans:
+                    print('#%-7u: idTestBox=%-3u tsCreated=%s tsDone=%s'
+                          % (oTestSet.idTestSet, oTestSet.idTestBox, oTestSet.tsCreated, oTestSet.tsDone));
+                    oLogic.completeAsAbandoned(oTestSet.idTestSet);
+                print('Committing...');
+                oDb.commit();
+            else:
+                for oTestSet in aoOrphans:
+                    print('#%-7u: idTestBox=%-3u tsCreated=%s tsDone=%s'
+                          % (oTestSet.idTestSet, oTestSet.idTestBox, oTestSet.tsCreated, oTestSet.tsDone));
+                print('Not completing any testsets without seeing the --just-do-it option.');
+        else:
+            print('No orphaned test sets.\n');
+        return 0;
+
+
+if __name__ == '__main__':
+    sys.exit(CloseOrphanedTestSets().main())
+
diff --git a/src/VBox/ValidationKit/testmanager/batch/del_build.py b/src/VBox/ValidationKit/testmanager/batch/del_build.py
new file mode 100755
index 00000000..252eb59d
--- /dev/null
+++ b/src/VBox/ValidationKit/testmanager/batch/del_build.py
@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# $Id: del_build.py $
+# pylint: disable=C0301
+
+"""
+Interface used by the tinderbox server side software to mark build binaries
+deleted.
+"""
+
+from __future__ import print_function;
+
+__copyright__ = \
+"""
+Copyright (C) 2012-2019 Oracle Corporation
+
+This file is part of VirtualBox Open Source Edition (OSE), as
+available from http://www.virtualbox.org. This file is free software;
+you can redistribute it and/or modify it under the terms of the GNU
+General Public License (GPL) as published by the Free Software
+Foundation, in version 2 as it comes in the "COPYING" file of the
+VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+
+The contents of this file may alternatively be used under the terms
+of the Common Development and Distribution License Version 1.0
+(CDDL) only, as it comes in the "COPYING.CDDL" file of the
+VirtualBox OSE distribution, in which case the provisions of the
+CDDL are applicable instead of those of the GPL.
+
+You may elect to license modified versions of this file under the
+terms and conditions of either the GPL or the CDDL or both.
+"""
+__version__ = "$Revision: 127855 $"
+
+# Standard python imports
+import sys
+import os
+from optparse import OptionParser;  # pylint: disable=deprecated-module
+
+# Add Test Manager's modules path
+g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.append(g_ksTestManagerDir)
+
+# Test Manager imports
+from testmanager.core.db    import TMDatabaseConnection
+from testmanager.core.build import BuildLogic
+
+
+def markBuildsDeleted():
+    """
+    Marks the builds using the specified binaries as deleted.
+    """
+
+    oParser = OptionParser()
+    oParser.add_option('-q', '--quiet', dest='fQuiet', action='store_true',
+                       help='Quiet execution');
+
+    (oConfig, asArgs) = oParser.parse_args()
+    if not asArgs:
+        if not oConfig.fQuiet:
+            sys.stderr.write('syntax error: No builds binaries specified\n');
+        return 1;
+
+
+    oDb = TMDatabaseConnection()
+    oLogic = BuildLogic(oDb)
+
+    for sBuildBin in asArgs:
+        try:
+            cBuilds = oLogic.markDeletedByBinaries(sBuildBin, fCommit = True)
+        except:
+            if oConfig.fQuiet:
+                sys.exit(1);
+            raise;
+        else:
+            if not oConfig.fQuiet:
+                print("del_build.py: Marked %u builds associated with '%s' as deleted." % (cBuilds, sBuildBin,));
+
+    oDb.close()
+    return 0;
+
+if __name__ == '__main__':
+    sys.exit(markBuildsDeleted())
+
diff --git a/src/VBox/ValidationKit/testmanager/batch/filearchiver.py b/src/VBox/ValidationKit/testmanager/batch/filearchiver.py
new file mode 100755
index 00000000..9c6bb8e8
--- /dev/null
+++ b/src/VBox/ValidationKit/testmanager/batch/filearchiver.py
@@ -0,0 +1,275 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# $Id: filearchiver.py $
+# pylint: disable=C0301
+
+"""
+A cronjob that compresses logs and other files, moving them to the
+g_ksZipFileAreaRootDir storage area.
+"""
+
+from __future__ import print_function;
+
+__copyright__ = \
+"""
+Copyright (C) 2012-2019 Oracle Corporation
+
+This file is part of VirtualBox Open Source Edition (OSE), as
+available from http://www.virtualbox.org. This file is free software;
+you can redistribute it and/or modify it under the terms of the GNU
+General Public License (GPL) as published by the Free Software
+Foundation, in version 2 as it comes in the "COPYING" file of the
+VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+
+The contents of this file may alternatively be used under the terms
+of the Common Development and Distribution License Version 1.0
+(CDDL) only, as it comes in the "COPYING.CDDL" file of the
+VirtualBox OSE distribution, in which case the provisions of the
+CDDL are applicable instead of those of the GPL.
+
+You may elect to license modified versions of this file under the
+terms and conditions of either the GPL or the CDDL or both.
+"""
+__version__ = "$Revision: 127855 $"
+
+# Standard python imports
+import sys
+import os
+from optparse import OptionParser;  # pylint: disable=deprecated-module
+import time;
+import zipfile;
+
+# Add Test Manager's modules path
+g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.append(g_ksTestManagerDir)
+
+# Test Manager imports
+from common                     import utils;
+from testmanager                import config;
+from testmanager.core.db        import TMDatabaseConnection;
+from testmanager.core.testset   import TestSetData, TestSetLogic;
+
+
+
+class FileArchiverBatchJob(object): # pylint: disable=R0903
+    """
+    Log+files comp
+    """
+
+    def __init__(self, oOptions):
+        """
+        Parse command line
+        """
+        self.fVerbose      = oOptions.fVerbose;
+        self.sSrcDir       = config.g_ksFileAreaRootDir;
+        self.sDstDir       = config.g_ksZipFileAreaRootDir;
+        #self.oTestSetLogic = TestSetLogic(TMDatabaseConnection(self.dprint if self.fVerbose else None));
+        self.oTestSetLogic = TestSetLogic(TMDatabaseConnection(None));
+        self.fDryRun       = oOptions.fDryRun;
+
+    def dprint(self, sText):
+        """ Verbose output. """
+        if self.fVerbose:
+            print(sText);
+        return True;
+
+    def warning(self, sText):
+        """Prints a warning."""
+        print(sText);
+        return True;
+
+    def _processTestSet(self, idTestSet, asFiles, sCurDir):
+        """
+        Worker for processDir.
+        Same return codes as processDir.
+        """
+
+        sBaseFilename = os.path.join(sCurDir, 'TestSet-%d' % (idTestSet,));
+        if sBaseFilename[0:2] == ('.' + os.path.sep):
+            sBaseFilename = sBaseFilename[2:];
+        sSrcFileBase = os.path.join(self.sSrcDir, sBaseFilename + '-');
+
+        #
+        # Skip the file if the test set is still running.
+        # But delete them if the testset is not found.
+        #
+        oTestSet = self.oTestSetLogic.tryFetch(idTestSet);
+        if oTestSet is not None and sBaseFilename != oTestSet.sBaseFilename:
+            self.warning('TestSet %d: Deleting because sBaseFilename differs: "%s" (disk) vs "%s" (db)' \
+                         % (idTestSet, sBaseFilename, oTestSet.sBaseFilename,));
+            oTestSet = None;
+
+        if oTestSet is not None:
+            if oTestSet.enmStatus == TestSetData.ksTestStatus_Running:
+                self.dprint('Skipping test set #%d, still running' % (idTestSet,));
+                return True;
+
+            #
+            # If we have a zip file already, don't try recreate it as we might
+            # have had trouble removing the source files.
+            #
+            sDstDirPath = os.path.join(self.sDstDir, sCurDir);
+            sZipFileNm  = os.path.join(sDstDirPath, 'TestSet-%d.zip' % (idTestSet,));
+            if not os.path.exists(sZipFileNm):
+                #
+                # Create zip file with all testset files as members.
+                #
+                self.dprint('TestSet %d: Creating %s...' % (idTestSet, sZipFileNm,));
+                if not self.fDryRun:
+
+                    if not os.path.exists(sDstDirPath):
+                        os.makedirs(sDstDirPath, 0o755);
+
+                    utils.noxcptDeleteFile(sZipFileNm + '.tmp');
+                    oZipFile = zipfile.ZipFile(sZipFileNm + '.tmp', 'w', zipfile.ZIP_DEFLATED, allowZip64 = True);
+
+                    for sFile in asFiles:
+                        sSuff = os.path.splitext(sFile)[1];
+                        if sSuff in [ '.png', '.webm', '.gz', '.bz2', '.zip', '.mov', '.avi', '.mpg', '.gif', '.jpg' ]:
+                            ## @todo Consider storing these files outside the zip if they are a little largish.
+                            self.dprint('TestSet %d: Storing   %s...' % (idTestSet, sFile));
+                            oZipFile.write(sSrcFileBase + sFile, sFile, zipfile.ZIP_STORED);
+                        else:
+                            self.dprint('TestSet %d: Deflating %s...' % (idTestSet, sFile));
+                            oZipFile.write(sSrcFileBase + sFile, sFile, zipfile.ZIP_DEFLATED);
+
+                    oZipFile.close();
+
+                    #
+                    # .zip.tmp -> .zip.
+                    #
+                    utils.noxcptDeleteFile(sZipFileNm);
+                    os.rename(sZipFileNm + '.tmp', sZipFileNm);
+
+                #else: Dry run.
+            else:
+                self.dprint('TestSet %d: zip file exists already (%s)' % (idTestSet, sZipFileNm,));
+
+        #
+        # Delete the files.
+        #
+        fRc = True;
+        if self.fVerbose:
+            self.dprint('TestSet %d: deleting file: %s' % (idTestSet, asFiles));
+        if not self.fDryRun:
+            for sFile in asFiles:
+                if utils.noxcptDeleteFile(sSrcFileBase + sFile) is False:
+                    self.warning('TestSet %d: Failed to delete "%s" (%s)' % (idTestSet, sFile, sSrcFileBase + sFile,));
+                    fRc = False;
+
+        return fRc;
+
+
+    def processDir(self, sCurDir):
+        """
+        Process the given directory (relative to sSrcDir and sDstDir).
+        Returns success indicator.
+        """
+        if self.fVerbose:
+            self.dprint('processDir: %s' % (sCurDir,));
+
+        #
+        # Sift thought the directory content, collecting subdirectories and
+        # sort relevant files by test set.
+        # Generally there will either be subdirs or there will be files.
+        #
+        asSubDirs = [];
+        dTestSets = {};
+        sCurPath = os.path.abspath(os.path.join(self.sSrcDir, sCurDir));
+        for sFile in os.listdir(sCurPath):
+            if os.path.isdir(os.path.join(sCurPath, sFile)):
+                if sFile not in [ '.', '..' ]:
+                    asSubDirs.append(sFile);
+            elif sFile.startswith('TestSet-'):
+                # Parse the file name. ASSUMES 'TestSet-%d-filename' format.
+                iSlash1 = sFile.find('-');
+                iSlash2 = sFile.find('-', iSlash1 + 1);
+                if iSlash2 <= iSlash1:
+                    self.warning('Bad filename (1): "%s"' % (sFile,));
+                    continue;
+
+                try:    idTestSet = int(sFile[(iSlash1 + 1):iSlash2]);
+                except:
+                    self.warning('Bad filename (2): "%s"' % (sFile,));
+                    if self.fVerbose:
+                        self.dprint('\n'.join(utils.getXcptInfo(4)));
+                    continue;
+
+                if idTestSet <= 0:
+                    self.warning('Bad filename (3): "%s"' % (sFile,));
+                    continue;
+
+                if iSlash2 + 2 >= len(sFile):
+                    self.warning('Bad filename (4): "%s"' % (sFile,));
+                    continue;
+                sName = sFile[(iSlash2 + 1):];
+
+                # Add it.
+                if idTestSet not in dTestSets:
+                    dTestSets[idTestSet] = [];
+                asTestSet = dTestSets[idTestSet];
+                asTestSet.append(sName);
+
+        #
+        # Test sets.
+        #
+        fRc = True;
+        for idTestSet in dTestSets:
+            try:
+                if self._processTestSet(idTestSet, dTestSets[idTestSet], sCurDir) is not True:
+                    fRc = False;
+            except:
+                self.warning('TestSet %d: Exception in _processTestSet:\n%s' % (idTestSet, '\n'.join(utils.getXcptInfo()),));
+                fRc = False;
+
+        #
+        # Sub dirs.
+        #
+        for sSubDir in asSubDirs:
+            if self.processDir(os.path.join(sCurDir, sSubDir)) is not True:
+                fRc = False;
+
+        #
+        # Try Remove the directory iff it's not '.' and it's been unmodified
+        # for the last 24h (race protection).
+        #
+        if sCurDir != '.':
+            try:
+                fpModTime = float(os.path.getmtime(sCurPath));
+                if fpModTime + (24*3600) <= time.time():
+                    if utils.noxcptRmDir(sCurPath) is True:
+                        self.dprint('Removed "%s".' % (sCurPath,));
+            except:
+                pass;
+
+        return fRc;
+
+    @staticmethod
+    def main():
+        """ C-style main(). """
+        #
+        # Parse options.
+        #
+        oParser = OptionParser();
+        oParser.add_option('-v', '--verbose', dest = 'fVerbose', action = 'store_true',  default = False,
+                           help = 'Verbose output.');
+        oParser.add_option('-q', '--quiet',   dest = 'fVerbose', action = 'store_false', default = False,
+                           help = 'Quiet operation.');
+        oParser.add_option('-d', '--dry-run', dest = 'fDryRun',  action = 'store_true',  default = False,
+                           help = 'Dry run, do not make any changes.');
+        (oOptions, asArgs) = oParser.parse_args()
+        if asArgs != []:
+            oParser.print_help();
+            return 1;
+
+        #
+        # Do the work.
+        #
+        oBatchJob = FileArchiverBatchJob(oOptions);
+        fRc = oBatchJob.processDir('.');
+        return 0 if fRc is True else 1;
+
+if __name__ == '__main__':
+    sys.exit(FileArchiverBatchJob.main());
+
diff --git a/src/VBox/ValidationKit/testmanager/batch/regen_sched_queues.py b/src/VBox/ValidationKit/testmanager/batch/regen_sched_queues.py
new file mode 100755
index 00000000..542854bd
--- /dev/null
+++ b/src/VBox/ValidationKit/testmanager/batch/regen_sched_queues.py
@@ -0,0 +1,122 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# $Id: regen_sched_queues.py $
+# pylint: disable=C0301
+
+"""
+Interface used by the admin to regenerate scheduling queues.
+"""
+
+from __future__ import print_function;
+
+__copyright__ = \
+"""
+Copyright (C) 2012-2019 Oracle Corporation
+
+This file is part of VirtualBox Open Source Edition (OSE), as
+available from http://www.virtualbox.org. This file is free software;
+you can redistribute it and/or modify it under the terms of the GNU
+General Public License (GPL) as published by the Free Software
+Foundation, in version 2 as it comes in the "COPYING" file of the
+VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+
+The contents of this file may alternatively be used under the terms
+of the Common Development and Distribution License Version 1.0
+(CDDL) only, as it comes in the "COPYING.CDDL" file of the
+VirtualBox OSE distribution, in which case the provisions of the
+CDDL are applicable instead of those of the GPL.
+
+You may elect to license modified versions of this file under the
+terms and conditions of either the GPL or the CDDL or both.
+"""
+__version__ = "$Revision: 127855 $"
+
+# Standard python imports
+import sys;
+import os;
+from optparse import OptionParser;  # pylint: disable=deprecated-module
+
+# Add Test Manager's modules path
+g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))));
+sys.path.append(g_ksTestManagerDir);
+
+# Test Manager imports
+from testmanager.core.db            import TMDatabaseConnection;
+from testmanager.core.schedulerbase import SchedulerBase;
+from testmanager.core.schedgroup    import SchedGroupLogic;
+
+
+
+class RegenSchedQueues(object): # pylint: disable=R0903
+    """
+    Regenerates all the scheduling queues.
+    """
+
+    def __init__(self):
+        """
+        Parse command line.
+        """
+
+        oParser = OptionParser();
+        oParser.add_option('-q', '--quiet', dest = 'fQuiet', action = 'store_true', default = False,
+                           help = 'Quiet execution');
+        oParser.add_option('-u', '--uid', dest = 'uid', action = 'store', type = 'int', default = 1,
+                           help = 'User ID to accredit with this job');
+        oParser.add_option('--profile', dest = 'fProfile', action = 'store_true', default = False,
+                           help = 'User ID to accredit with this job');
+
+        (self.oConfig, _) = oParser.parse_args();
+
+
+    def doIt(self):
+        """
+        Does the job.
+        """
+        oDb = TMDatabaseConnection();
+
+        aoGroups = SchedGroupLogic(oDb).getAll();
+        iRc = 0;
+        for oGroup in aoGroups:
+            if not self.oConfig.fQuiet:
+                print('%s (ID %#d):' % (oGroup.sName, oGroup.idSchedGroup,));
+            try:
+                (aoErrors, asMessages) = SchedulerBase.recreateQueue(oDb, self.oConfig.uid, oGroup.idSchedGroup, 2);
+            except Exception as oXcpt:
+                oDb.rollback();
+                print('  !!Hit exception processing "%s": %s' % (oGroup.sName, oXcpt,));
+            else:
+                if not aoErrors:
+                    if not self.oConfig.fQuiet:
+                        print('  Successfully regenerated.');
+                else:
+                    iRc = 1;
+                    print('  %d errors:' % (len(aoErrors,)));
+                    for oError in aoErrors:
+                        if oError[1]  is None:
+                            print('  !!%s' % (oError[0],));
+                        else:
+                            print('  !!%s (%s)' % (oError[0], oError[1]));
+                if asMessages and not self.oConfig.fQuiet:
+                    print('  %d messages:' % (len(asMessages),));
+                    for sMsg in asMessages:
+                        print('  ##%s' % (sMsg,));
+        return iRc;
+
+    @staticmethod
+    def main():
+        """ Main function. """
+        oMain = RegenSchedQueues();
+        if oMain.oConfig.fProfile is not True:
+            iRc = oMain.doIt();
+        else:
+            import cProfile;
+            oProfiler = cProfile.Profile();
+            iRc = oProfiler.runcall(oMain.doIt);
+            oProfiler.print_stats(sort = 'time');
+            oProfiler = None;
+        return iRc;
+
+if __name__ == '__main__':
+    sys.exit(RegenSchedQueues().main());
+
diff --git a/src/VBox/ValidationKit/testmanager/batch/vcs_import.py b/src/VBox/ValidationKit/testmanager/batch/vcs_import.py
new file mode 100755
index 00000000..c840bf3e
--- /dev/null
+++ b/src/VBox/ValidationKit/testmanager/batch/vcs_import.py
@@ -0,0 +1,144 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# $Id: vcs_import.py $
+# pylint: disable=C0301
+
+"""
+Cron job for importing revision history for a repository.
+"""
+
+from __future__ import print_function;
+
+__copyright__ = \
+"""
+Copyright (C) 2012-2019 Oracle Corporation
+
+This file is part of VirtualBox Open Source Edition (OSE), as
+available from http://www.virtualbox.org. This file is free software;
+you can redistribute it and/or modify it under the terms of the GNU
+General Public License (GPL) as published by the Free Software
+Foundation, in version 2 as it comes in the "COPYING" file of the
+VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+
+The contents of this file may alternatively be used under the terms
+of the Common Development and Distribution License Version 1.0
+(CDDL) only, as it comes in the "COPYING.CDDL" file of the
+VirtualBox OSE distribution, in which case the provisions of the
+CDDL are applicable instead of those of the GPL.
+
+You may elect to license modified versions of this file under the
+terms and conditions of either the GPL or the CDDL or both.
+"""
+__version__ = "$Revision: 127855 $"
+
+# Standard python imports
+import sys;
+import os;
+from optparse import OptionParser; # pylint: disable=deprecated-module
+import xml.etree.ElementTree as ET;
+
+# Add Test Manager's modules path
+g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))));
+sys.path.append(g_ksTestManagerDir);
+
+# Test Manager imports
+from testmanager.core.db            import TMDatabaseConnection;
+from testmanager.core.vcsrevisions  import VcsRevisionData, VcsRevisionLogic;
+from common                         import utils;
+
+class VcsImport(object): # pylint: disable=R0903
+    """
+    Imports revision history from a VSC into the Test Manager database.
+    """
+
+    def __init__(self):
+        """
+        Parse command line.
+        """
+
+        oParser = OptionParser()
+        oParser.add_option('-e', '--extra-option', dest = 'asExtraOptions', action = 'append',
+                           help = 'Adds a extra option to the command retrieving the log.');
+        oParser.add_option('-f', '--full', dest = 'fFull', action = 'store_true',
+                           help = 'Full revision history import.');
+        oParser.add_option('-q', '--quiet', dest = 'fQuiet', action = 'store_true',
+                           help = 'Quiet execution');
+        oParser.add_option('-R', '--repository', dest = 'sRepository', metavar = '<repository>',
+                           help = 'Version control repository name.');
+        oParser.add_option('-s', '--start-revision', dest = 'iStartRevision', metavar = 'start-revision',
+                           type = "int", default = 0,
+                           help = 'The revision to start at when doing a full import.');
+        oParser.add_option('-t', '--type', dest = 'sType', metavar = '<type>',
+                           help = 'The VCS type (default: svn)', choices = [ 'svn', ], default = 'svn');
+        oParser.add_option('-u', '--url', dest = 'sUrl', metavar = '<url>',
+                           help = 'The VCS URL');
+
+        (self.oConfig, _) = oParser.parse_args();
+
+        # Check command line
+        asMissing = [];
+        if self.oConfig.sUrl is None:               asMissing.append('--url');
+        if self.oConfig.sRepository is None:        asMissing.append('--repository');
+        if asMissing:
+            sys.stderr.write('syntax error: Missing: %s\n' % (asMissing,));
+            sys.exit(1);
+
+        assert self.oConfig.sType == 'svn';
+
+    def main(self):
+        """
+        Main function.
+        """
+        oDb = TMDatabaseConnection();
+        oLogic = VcsRevisionLogic(oDb);
+
+        # Where to start.
+        iStartRev = 0;
+        if not self.oConfig.fFull:
+            iStartRev = oLogic.getLastRevision(self.oConfig.sRepository);
+        if iStartRev == 0:
+            iStartRev = self.oConfig.iStartRevision;
+
+        # Construct a command line.
+        os.environ['LC_ALL'] = 'en_US.utf-8';
+        asArgs = [
+            'svn',
+            'log',
+            '--xml',
+            '--revision', str(iStartRev) + ':HEAD',
+        ];
+        if self.oConfig.asExtraOptions is not None:
+            asArgs.extend(self.oConfig.asExtraOptions);
+        asArgs.append(self.oConfig.sUrl);
+        if not self.oConfig.fQuiet:
+            print('Executing: %s' % (asArgs,));
+        sLogXml = utils.processOutputChecked(asArgs);
+
+        # Parse the XML and add the entries to the database.
+        oParser = ET.XMLParser(target = ET.TreeBuilder(), encoding = 'utf-8');
+        oParser.feed(sLogXml.encode('utf-8')); # does its own decoding and processOutputChecked always gives us decoded utf-8 now.
+        oRoot = oParser.close();
+
+        for oLogEntry in oRoot.findall('logentry'):
+            iRevision = int(oLogEntry.get('revision'));
+            sAuthor  = oLogEntry.findtext('author').strip();
+            sDate    = oLogEntry.findtext('date').strip();
+            sMessage = oLogEntry.findtext('msg', '').strip();
+            if sMessage == '':
+                sMessage = ' ';
+            elif len(sMessage) > VcsRevisionData.kcchMax_sMessage:
+                sMessage = sMessage[:VcsRevisionData.kcchMax_sMessage - 4] + ' ...';
+            if not self.oConfig.fQuiet:
+                utils.printOut(u'sDate=%s iRev=%u sAuthor=%s sMsg[%s]=%s'
+                               % (sDate, iRevision, sAuthor, type(sMessage).__name__, sMessage));
+            oData = VcsRevisionData().initFromValues(self.oConfig.sRepository, iRevision, sDate, sAuthor, sMessage);
+            oLogic.addVcsRevision(oData);
+        oDb.commit();
+
+        oDb.close();
+        return 0;
+
+if __name__ == '__main__':
+    sys.exit(VcsImport().main());
+
diff --git a/src/VBox/ValidationKit/testmanager/batch/virtual_test_sheriff.py b/src/VBox/ValidationKit/testmanager/batch/virtual_test_sheriff.py
new file mode 100755
index 00000000..3f8f2ee3
--- /dev/null
+++ b/src/VBox/ValidationKit/testmanager/batch/virtual_test_sheriff.py
@@ -0,0 +1,1466 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# $Id: virtual_test_sheriff.py $
+# pylint: disable=C0301
+
+"""
+Virtual Test Sheriff.
+
+Duties:
+    - Try to a assign failure reasons to recently failed tests.
+    - Reboot or disable bad test boxes.
+
+"""
+
+from __future__ import print_function;
+
+__copyright__ = \
+"""
+Copyright (C) 2012-2019 Oracle Corporation
+
+This file is part of VirtualBox Open Source Edition (OSE), as
+available from http://www.virtualbox.org. This file is free software;
+you can redistribute it and/or modify it under the terms of the GNU
+General Public License (GPL) as published by the Free Software
+Foundation, in version 2 as it comes in the "COPYING" file of the
+VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+
+The contents of this file may alternatively be used under the terms
+of the Common Development and Distribution License Version 1.0
+(CDDL) only, as it comes in the "COPYING.CDDL" file of the
+VirtualBox OSE distribution, in which case the provisions of the
+CDDL are applicable instead of those of the GPL.
+
+You may elect to license modified versions of this file under the
+terms and conditions of either the GPL or the CDDL or both.
+"""
+__version__ = "$Revision: 127929 $"
+
+
+# Standard python imports
+import sys;
+import os;
+import hashlib;
+import subprocess;
+import smtplib
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+from email.utils import COMMASPACE
+
+if sys.version_info[0] >= 3:
+    from io       import StringIO as StringIO;      # pylint: disable=import-error,no-name-in-module
+else:
+    from StringIO import StringIO as StringIO;      # pylint: disable=import-error,no-name-in-module
+from optparse import OptionParser;                  # pylint: disable=deprecated-module
+from PIL import Image;                              # pylint: disable=import-error
+
+# Add Test Manager's modules path
+g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))));
+sys.path.append(g_ksTestManagerDir);
+
+# Test Manager imports
+from testmanager.core.db                    import TMDatabaseConnection;
+from testmanager.core.build                 import BuildDataEx;
+from testmanager.core.failurereason         import FailureReasonLogic;
+from testmanager.core.testbox               import TestBoxLogic, TestBoxData;
+from testmanager.core.testcase              import TestCaseDataEx;
+from testmanager.core.testgroup             import TestGroupData;
+from testmanager.core.testset               import TestSetLogic, TestSetData;
+from testmanager.core.testresults           import TestResultLogic, TestResultFileData;
+from testmanager.core.testresultfailures    import TestResultFailureLogic, TestResultFailureData;
+from testmanager.core.useraccount           import UserAccountLogic;
+from testmanager.config                     import g_ksSmtpHost, g_kcSmtpPort, g_ksAlertFrom, \
+                                                   g_ksAlertSubject, g_asAlertList, g_ksLomPassword;
+
+# Python 3 hacks:
+if sys.version_info[0] >= 3:
+    xrange = range; # pylint: disable=redefined-builtin,invalid-name
+
+
+class VirtualTestSheriffCaseFile(object):
+    """
+    A failure investigation case file.
+
+    """
+
+
+    ## Max log file we'll read into memory. (256 MB)
+    kcbMaxLogRead = 0x10000000;
+
+    def __init__(self, oSheriff, oTestSet, oTree, oBuild, oTestBox, oTestGroup, oTestCase):
+        self.oSheriff       = oSheriff;
+        self.oTestSet       = oTestSet;     # TestSetData
+        self.oTree          = oTree;        # TestResultDataEx
+        self.oBuild         = oBuild;       # BuildDataEx
+        self.oTestBox       = oTestBox;     # TestBoxData
+        self.oTestGroup     = oTestGroup;   # TestGroupData
+        self.oTestCase      = oTestCase;    # TestCaseDataEx
+        self.sMainLog       = '';           # The main log file.  Empty string if not accessible.
+
+        # Generate a case file name.
+        self.sName          = '#%u: %s' % (self.oTestSet.idTestSet, self.oTestCase.sName,)
+        self.sLongName      = '#%u: "%s" on "%s" running %s %s (%s), "%s" by %s, using %s %s %s r%u'  \
+                            % ( self.oTestSet.idTestSet,
+                                self.oTestCase.sName,
+                                self.oTestBox.sName,
+                                self.oTestBox.sOs,
+                                self.oTestBox.sOsVersion,
+                                self.oTestBox.sCpuArch,
+                                self.oTestBox.sCpuName,
+                                self.oTestBox.sCpuVendor,
+                                self.oBuild.oCat.sProduct,
+                                self.oBuild.oCat.sBranch,
+                                self.oBuild.oCat.sType,
+                                self.oBuild.iRevision, );
+
+        # Investigation notes.
+        self.tReason                = None; # None or one of the ktReason_XXX constants.
+        self.dReasonForResultId     = {};   # Reason assignments indexed by idTestResult.
+        self.dCommentForResultId    = {};   # Comment assignments indexed by idTestResult.
+
+    #
+    # Reason.
+    #
+
+    def noteReason(self, tReason):
+        """ Notes down a possible reason. """
+        self.oSheriff.dprint(u'noteReason: %s -> %s' % (self.tReason, tReason,));
+        self.tReason = tReason;
+        return True;
+
+    def noteReasonForId(self, tReason, idTestResult, sComment = None):
+        """ Notes down a possible reason for a specific test result. """
+        self.oSheriff.dprint(u'noteReasonForId: %u: %s -> %s%s'
+                             % (idTestResult, self.dReasonForResultId.get(idTestResult, None), tReason,
+                                (u' (%s)' % (sComment,)) if sComment is not None else ''));
+        self.dReasonForResultId[idTestResult] = tReason;
+        if sComment is not None:
+            self.dCommentForResultId[idTestResult] = sComment;
+        return True;
+
+
+    #
+    # Test classification.
+    #
+
+    def isVBoxTest(self):
+        """ Test classification: VirtualBox (using the build) """
+        return self.oBuild.oCat.sProduct.lower() in [ 'virtualbox', 'vbox' ];
+
+    def isVBoxUnitTest(self):
+        """ Test case classification: The unit test doing all our testcase/*.cpp stuff. """
+        return self.isVBoxTest() \
+           and (self.oTestCase.sName.lower() == 'unit tests' or self.oTestCase.sName.lower() == 'misc: unit tests');
+
+    def isVBoxInstallTest(self):
+        """ Test case classification: VirtualBox Guest installation test. """
+        return self.isVBoxTest() \
+           and self.oTestCase.sName.lower().startswith('install:');
+
+    def isVBoxUSBTest(self):
+        """ Test case classification: VirtualBox USB test. """
+        return self.isVBoxTest() \
+           and self.oTestCase.sName.lower().startswith('usb:');
+
+    def isVBoxStorageTest(self):
+        """ Test case classification: VirtualBox Storage test. """
+        return self.isVBoxTest() \
+           and self.oTestCase.sName.lower().startswith('storage:');
+
+    def isVBoxGAsTest(self):
+        """ Test case classification: VirtualBox Guest Additions test. """
+        return self.isVBoxTest() \
+           and self.oTestCase.sName.lower().startswith('ga\'s tests');
+
+    def isVBoxAPITest(self):
+        """ Test case classification: VirtualBox API test. """
+        return self.isVBoxTest() \
+           and self.oTestCase.sName.lower().startswith('api:');
+
+    def isVBoxBenchmarkTest(self):
+        """ Test case classification: VirtualBox Benchmark test. """
+        return self.isVBoxTest() \
+           and self.oTestCase.sName.lower().startswith('benchmark:');
+
+    def isVBoxSmokeTest(self):
+        """ Test case classification: Smoke test. """
+        return self.isVBoxTest() \
+           and self.oTestCase.sName.lower().startswith('smoketest');
+
+
+    #
+    # Utility methods.
+    #
+
+    def getMainLog(self):
+        """
+        Tries to read the main log file since this will be the first source of information.
+        """
+        if self.sMainLog:
+            return self.sMainLog;
+        (oFile, oSizeOrError, _) = self.oTestSet.openFile('main.log', 'rb');
+        if oFile is not None:
+            try:
+                self.sMainLog = oFile.read(min(self.kcbMaxLogRead, oSizeOrError)).decode('utf-8', 'replace');
+            except Exception as oXcpt:
+                self.oSheriff.vprint(u'Error reading main log file: %s' % (oXcpt,))
+                self.sMainLog = '';
+        else:
+            self.oSheriff.vprint(u'Error opening main log file: %s' % (oSizeOrError,));
+        return self.sMainLog;
+
+    def getLogFile(self, oFile):
+        """
+        Tries to read the given file as a utf-8 log file.
+        oFile is a TestFileDataEx instance.
+        Returns empty string if problems opening or reading the file.
+        """
+        sContent = '';
+        (oFile, oSizeOrError, _) = self.oTestSet.openFile(oFile.sFile, 'rb');
+        if oFile is not None:
+            try:
+                sContent = oFile.read(min(self.kcbMaxLogRead, oSizeOrError)).decode('utf-8', 'replace');
+            except Exception as oXcpt:
+                self.oSheriff.vprint(u'Error reading the "%s" log file: %s' % (oFile.sFile, oXcpt,))
+        else:
+            self.oSheriff.vprint(u'Error opening the "%s" log file: %s' % (oFile.sFile, oSizeOrError,));
+        return sContent;
+
+    def getScreenshotSha256(self, oFile):
+        """
+        Tries to read the given screenshot file, uncompress it, and do SHA-2
+        on the raw pixels.
+        Returns SHA-2 digest string on success, None on failure.
+        """
+        (oImgFile, _, _) = self.oTestSet.openFile(oFile.sFile, 'rb');
+        try:
+            abImageFile = oImgFile.read();
+        except Exception as oXcpt:
+            self.oSheriff.vprint(u'Error reading the "%s" image file: %s' % (oFile.sFile, oXcpt,))
+        else:
+            try:
+                oImage = Image.open(StringIO(abImageFile));
+            except Exception as oXcpt:
+                self.oSheriff.vprint(u'Error opening the "%s" image bytes using PIL.Image.open: %s' % (oFile.sFile, oXcpt,))
+            else:
+                try:
+                    oHash = hashlib.sha256();
+                    oHash.update(oImage.tostring());
+                except Exception as oXcpt:
+                    self.oSheriff.vprint(u'Error hashing the uncompressed image bytes for "%s": %s' % (oFile.sFile, oXcpt,))
+                else:
+                    return oHash.hexdigest();
+        return None;
+
+
+
+    def isSingleTestFailure(self):
+        """
+        Figure out if this is a single test failing or if it's one of the
+        more complicated ones.
+        """
+        if self.oTree.cErrors == 1:
+            return True;
+        if self.oTree.deepCountErrorContributers() <= 1:
+            return True;
+        return False;
+
+
+
+class VirtualTestSheriff(object): # pylint: disable=R0903
+    """
+    Add build info into Test Manager database.
+    """
+
+    ## The user account for the virtual sheriff.
+    ksLoginName = 'vsheriff';
+
+    def __init__(self):
+        """
+        Parse command line.
+        """
+        self.oDb                     = None;
+        self.tsNow                   = None;
+        self.oTestResultLogic        = None;
+        self.oTestSetLogic           = None;
+        self.oFailureReasonLogic     = None;    # FailureReasonLogic;
+        self.oTestResultFailureLogic = None;    # TestResultFailureLogic
+        self.oLogin                  = None;
+        self.uidSelf                 = -1;
+        self.oLogFile                = None;
+        self.asBsodReasons           = [];
+        self.asUnitTestReasons       = [];
+
+        oParser = OptionParser();
+        oParser.add_option('--start-hours-ago', dest = 'cStartHoursAgo', metavar = '<hours>', default = 0, type = 'int',
+                           help = 'When to start specified as hours relative to current time.  Defauls is right now.', );
+        oParser.add_option('--hours-period', dest = 'cHoursBack', metavar = '<period-in-hours>', default = 2, type = 'int',
+                           help = 'Work period specified in hours.  Defauls is 2 hours.');
+        oParser.add_option('--real-run-back', dest = 'fRealRun', action = 'store_true', default = False,
+                           help = 'Whether to commit the findings to the database. Default is a dry run.');
+        oParser.add_option('-q', '--quiet', dest = 'fQuiet', action = 'store_true', default = False,
+                           help = 'Quiet execution');
+        oParser.add_option('-l', '--log', dest = 'sLogFile', metavar = '<logfile>', default = None,
+                           help = 'Where to log messages.');
+        oParser.add_option('--debug', dest = 'fDebug', action = 'store_true', default = False,
+                           help = 'Enables debug mode.');
+
+        (self.oConfig, _) = oParser.parse_args();
+
+        if self.oConfig.sLogFile:
+            self.oLogFile = open(self.oConfig.sLogFile, "a");
+            self.oLogFile.write('VirtualTestSheriff: $Revision: 127929 $ \n');
+
+
+    def eprint(self, sText):
+        """
+        Prints error messages.
+        Returns 1 (for exit code usage.)
+        """
+        print('error: %s' % (sText,));
+        if self.oLogFile is not None:
+            self.oLogFile.write((u'error: %s\n' % (sText,)).encode('utf-8'));
+        return 1;
+
+    def dprint(self, sText):
+        """
+        Prints debug info.
+        """
+        if self.oConfig.fDebug:
+            if not self.oConfig.fQuiet:
+                print('debug: %s' % (sText, ));
+            if self.oLogFile is not None:
+                self.oLogFile.write((u'debug: %s\n' % (sText,)).encode('utf-8'));
+        return 0;
+
+    def vprint(self, sText):
+        """
+        Prints verbose info.
+        """
+        if not self.oConfig.fQuiet:
+            print('info: %s' % (sText,));
+        if self.oLogFile is not None:
+            self.oLogFile.write((u'info: %s\n' % (sText,)).encode('utf-8'));
+        return 0;
+
+    def getFailureReason(self, tReason):
+        """ Gets the failure reason object for tReason. """
+        return self.oFailureReasonLogic.cachedLookupByNameAndCategory(tReason[1], tReason[0]);
+
+    def selfCheck(self):
+        """ Does some self checks, looking up things we expect to be in the database and such. """
+        rcExit = 0;
+        for sAttr in dir(self.__class__):
+            if sAttr.startswith('ktReason_'):
+                tReason = getattr(self.__class__, sAttr);
+                oFailureReason = self.getFailureReason(tReason);
+                if oFailureReason is None:
+                    rcExit = self.eprint(u'Failed to find failure reason "%s" in category "%s" in the database!'
+                                         % (tReason[1], tReason[0],));
+
+        # Check the user account as well.
+        if self.oLogin is None:
+            oLogin = UserAccountLogic(self.oDb).tryFetchAccountByLoginName(VirtualTestSheriff.ksLoginName);
+            if oLogin is None:
+                rcExit = self.eprint(u'Cannot find my user account "%s"!' % (VirtualTestSheriff.ksLoginName,));
+        return rcExit;
+
+    def sendEmailAlert(self, uidAuthor, sBodyText):
+        """
+        Sends email alert.
+        """
+
+        # Get author email
+        self.oDb.execute('SELECT sEmail FROM Users WHERE uid=%s', (uidAuthor,));
+        sFrom = self.oDb.fetchOne();
+        if sFrom is not None:
+            sFrom = sFrom[0];
+        else:
+            sFrom = g_ksAlertFrom;
+
+        # Gather recipient list.
+        asEmailList = [];
+        for sUser in g_asAlertList:
+            self.oDb.execute('SELECT sEmail FROM Users WHERE sUsername=%s', (sUser,));
+            sEmail = self.oDb.fetchOne();
+            if sEmail:
+                asEmailList.append(sEmail[0]);
+        if not asEmailList:
+            return self.eprint('No email addresses to send alter to!');
+
+        # Compose the message.
+        oMsg = MIMEMultipart();
+        oMsg['From'] = sFrom;
+        oMsg['To'] = COMMASPACE.join(asEmailList);
+        oMsg['Subject'] = g_ksAlertSubject;
+        oMsg.attach(MIMEText(sBodyText, 'plain'))
+
+        # Try send it.
+        try:
+            oSMTP = smtplib.SMTP(g_ksSmtpHost, g_kcSmtpPort);
+            oSMTP.sendmail(sFrom, asEmailList, oMsg.as_string())
+            oSMTP.quit()
+        except smtplib.SMTPException as oXcpt:
+            return self.eprint('Failed to send mail: %s' % (oXcpt,));
+
+        return 0;
+
+    def badTestBoxManagement(self):
+        """
+        Looks for bad test boxes and first tries once to reboot them then disables them.
+        """
+        rcExit = 0;
+
+        #
+        # We skip this entirely if we're running in the past and not in harmless debug mode.
+        #
+        if    self.oConfig.cStartHoursAgo != 0 \
+          and (not self.oConfig.fDebug or self.oConfig.fRealRun):
+            return rcExit;
+        tsNow      = self.tsNow              if self.oConfig.fDebug else None;
+        cHoursBack = self.oConfig.cHoursBack if self.oConfig.fDebug else 2;
+        oTestBoxLogic = TestBoxLogic(self.oDb);
+
+        #
+        # Generate a list of failures reasons we consider bad-testbox behavior.
+        #
+        aidFailureReasons = [
+            self.getFailureReason(self.ktReason_Host_DriverNotUnloading).idFailureReason,
+            self.getFailureReason(self.ktReason_Host_DriverNotCompilable).idFailureReason,
+            self.getFailureReason(self.ktReason_Host_InstallationFailed).idFailureReason,
+        ];
+
+        #
+        # Get list of bad test boxes for given period and check them out individually.
+        #
+        aidBadTestBoxes = self.oTestSetLogic.fetchBadTestBoxIds(cHoursBack = cHoursBack, tsNow = tsNow,
+                                                                aidFailureReasons = aidFailureReasons);
+        for idTestBox in aidBadTestBoxes:
+            # Skip if the testbox is already disabled or has a pending reboot command.
+            try:
+                oTestBox = TestBoxData().initFromDbWithId(self.oDb, idTestBox);
+            except Exception as oXcpt:
+                rcExit = self.eprint('Failed to get data for test box #%u in badTestBoxManagement: %s' % (idTestBox, oXcpt,));
+                continue;
+            if not oTestBox.fEnabled:
+                self.dprint(u'badTestBoxManagement: Skipping test box #%u (%s) as it has been disabled already.'
+                            % ( idTestBox, oTestBox.sName, ));
+                continue;
+            if oTestBox.enmPendingCmd != TestBoxData.ksTestBoxCmd_None:
+                self.dprint(u'badTestBoxManagement: Skipping test box #%u (%s) as it has a command pending: %s'
+                            % ( idTestBox, oTestBox.sName, oTestBox.enmPendingCmd));
+                continue;
+
+            # Get the most recent testsets for this box (descending on tsDone) and see how bad it is.
+            aoSets  = self.oTestSetLogic.fetchSetsForTestBox(idTestBox, cHoursBack = cHoursBack, tsNow = tsNow);
+            cOkay      = 0;
+            cBad       = 0;
+            iFirstOkay = len(aoSets);
+            for iSet, oSet in enumerate(aoSets):
+                if oSet.enmStatus == TestSetData.ksTestStatus_BadTestBox:
+                    cBad += 1;
+                else:
+                    # Check for bad failure reasons.
+                    oFailure = None;
+                    if oSet.enmStatus in TestSetData.kasBadTestStatuses:
+                        oFailure = self.oTestResultFailureLogic.getById(oSet.idTestResult);
+                    if oFailure is not None and oFailure.idFailureReason in aidFailureReasons:
+                        cBad += 1;
+                    else:
+                        # This is an okay test result then.
+                        ## @todo maybe check the elapsed time here, it could still be a bad run?
+                        cOkay += 1;
+                        if iFirstOkay > iSet:
+                            iFirstOkay = iSet;
+                if iSet > 10:
+                    break;
+
+            # We react if there are two or more bad-testbox statuses at the head of the
+            # history and at least three in the last 10 results.
+            if iFirstOkay >= 2 and cBad > 2:
+                # Frank: For now don't reboot boxes automatically
+                if True or oTestBoxLogic.hasTestBoxRecentlyBeenRebooted(idTestBox, cHoursBack = cHoursBack, tsNow = tsNow):
+                    self.vprint(u'Disabling testbox #%u (%s) - iFirstOkay=%u cBad=%u cOkay=%u'
+                                % ( idTestBox, oTestBox.sName, iFirstOkay, cBad, cOkay));
+                    if self.oConfig.fRealRun is True:
+                        try:
+                            oTestBoxLogic.disableTestBox(idTestBox, self.uidSelf, fCommit = True,
+                                                         sComment = 'Automatically disabled (iFirstOkay=%u cBad=%u cOkay=%u)'
+                                                                  % (iFirstOkay, cBad, cOkay),);
+                        except Exception as oXcpt:
+                            rcExit = self.eprint(u'Error disabling testbox #%u (%u): %s\n' % (idTestBox, oTestBox.sName, oXcpt,));
+                else:
+                    self.vprint(u'Rebooting testbox #%u (%s) - iFirstOkay=%u cBad=%u cOkay=%u'
+                                % ( idTestBox, oTestBox.sName, iFirstOkay, cBad, cOkay));
+                    if self.oConfig.fRealRun is True:
+                        try:
+                            oTestBoxLogic.rebootTestBox(idTestBox, self.uidSelf, fCommit = True,
+                                                        sComment = 'Automatically rebooted (iFirstOkay=%u cBad=%u cOkay=%u)'
+                                                                 % (iFirstOkay, cBad, cOkay),);
+                        except Exception as oXcpt:
+                            rcExit = self.eprint(u'Error rebooting testbox #%u (%s): %s\n' % (idTestBox, oTestBox.sName, oXcpt,));
+            else:
+                self.dprint(u'badTestBoxManagement: #%u (%s) looks ok:  iFirstOkay=%u cBad=%u cOkay=%u'
+                            % ( idTestBox, oTestBox.sName, iFirstOkay, cBad, cOkay));
+        #
+        # Reset hanged testboxes
+        #
+        cStatusTimeoutMins = 10;
+
+        self.oDb.execute('SELECT idTestBox FROM TestBoxStatuses WHERE tsUpdated < (CURRENT_TIMESTAMP - interval \'%s minutes\')', (cStatusTimeoutMins,));
+        for idTestBox in self.oDb.fetchAll():
+            idTestBox = idTestBox[0];
+            try:
+                oTestBox = TestBoxData().initFromDbWithId(self.oDb, idTestBox);
+            except Exception as oXcpt:
+                rcExit = self.eprint('Failed to get data for test box #%u in badTestBoxManagement: %s' % (idTestBox, oXcpt,));
+                continue;
+            # Skip if the testbox is already disabled, already reset or there's no iLOM
+            if not oTestBox.fEnabled or oTestBox.ipLom is None or oTestBox.sComment is not None and oTestBox.sComment.find('Automatically reset') >= 0:
+                self.dprint(u'badTestBoxManagement: Skipping test box #%u (%s) as it has been disabled already.'
+                            % ( idTestBox, oTestBox.sName, ));
+                continue;
+            ## @todo get iLOM credentials from a table?
+            sCmd = 'sshpass -p%s ssh -oStrictHostKeyChecking=no root@%s show /SP && reset /SYS' % (g_ksLomPassword, oTestBox.ipLom,);
+            try:
+                oPs = subprocess.Popen(sCmd, stdout=subprocess.PIPE, shell=True);
+                sStdout = oPs.communicate()[0];
+                iRC = oPs.wait();
+
+                oTestBox.sComment = 'Automatically reset (iRC=%u sStdout=%s)' % (iRC, sStdout,);
+                oTestBoxLogic.editEntry(oTestBox, self.uidSelf, fCommit = True);
+
+                sComment = u'Reset testbox #%u (%s) - iRC=%u sStduot=%s' % ( idTestBox, oTestBox.sName, iRC, sStdout);
+                self.vprint(sComment);
+                self.sendEmailAlert(self.uidSelf, sComment);
+
+            except Exception as oXcpt:
+                rcExit = self.eprint(u'Error reseting testbox #%u (%s): %s\n' % (idTestBox, oTestBox.sName, oXcpt,));
+
+        return rcExit;
+
+
+    ## @name Failure reasons we know.
+    ## @{
+    ktReason_BSOD_Recovery                             = ( 'BSOD',              'Recovery' );
+    ktReason_BSOD_Automatic_Repair                     = ( 'BSOD',              'Automatic Repair' );
+    ktReason_BSOD_0000007F                             = ( 'BSOD',              '0x0000007F' );
+    ktReason_BSOD_000000D1                             = ( 'BSOD',              '0x000000D1' );
+    ktReason_BSOD_C0000225                             = ( 'BSOD',              '0xC0000225 (boot)' );
+    ktReason_Guru_Generic                              = ( 'Guru Meditations',  'Generic Guru Meditation' );
+    ktReason_Guru_VERR_IEM_INSTR_NOT_IMPLEMENTED       = ( 'Guru Meditations',  'VERR_IEM_INSTR_NOT_IMPLEMENTED' );
+    ktReason_Guru_VERR_IEM_ASPECT_NOT_IMPLEMENTED      = ( 'Guru Meditations',  'VERR_IEM_ASPECT_NOT_IMPLEMENTED' );
+    ktReason_Guru_VERR_TRPM_DONT_PANIC                 = ( 'Guru Meditations',  'VERR_TRPM_DONT_PANIC' );
+    ktReason_Guru_VERR_PGM_PHYS_PAGE_RESERVED          = ( 'Guru Meditations',  'VERR_PGM_PHYS_PAGE_RESERVED' );
+    ktReason_Guru_VERR_VMX_INVALID_GUEST_STATE         = ( 'Guru Meditations',  'VERR_VMX_INVALID_GUEST_STATE' );
+    ktReason_Guru_VINF_EM_TRIPLE_FAULT                 = ( 'Guru Meditations',  'VINF_EM_TRIPLE_FAULT' );
+    ktReason_Host_HostMemoryLow                        = ( 'Host',              'HostMemoryLow' );
+    ktReason_Host_DriverNotLoaded                      = ( 'Host',              'Driver not loaded' );
+    ktReason_Host_DriverNotUnloading                   = ( 'Host',              'Driver not unloading' );
+    ktReason_Host_DriverNotCompilable                  = ( 'Host',              'Driver not compilable' );
+    ktReason_Host_InstallationFailed                   = ( 'Host',              'Installation failed' );
+    ktReason_Host_NotSignedWithBuildCert               = ( 'Host',              'Not signed with build cert' );
+    ktReason_Host_DoubleFreeHeap                       = ( 'Host',              'Double free or corruption' );
+    ktReason_Host_LeftoverService                      = ( 'Host',              'Leftover service' );
+    ktReason_Host_Reboot_OSX_Watchdog_Timeout          = ( 'Host Reboot',       'OSX Watchdog Timeout' );
+    ktReason_Host_Modprobe_Failed                      = ( 'Host',              'Modprobe failed' );
+    ktReason_Host_Install_Hang                         = ( 'Host',              'Install hang' );
+    ktReason_Host_NetworkMisconfiguration              = ( 'Host',              'Network misconfiguration' );
+    ktReason_Networking_Nonexistent_host_nic           = ( 'Networking',        'Nonexistent host networking interface' );
+    ktReason_OSInstall_GRUB_hang                       = ( 'O/S Install',       'GRUB hang' );
+    ktReason_OSInstall_Udev_hang                       = ( 'O/S Install',       'udev hang' );
+    ktReason_OSInstall_Sata_no_BM                      = ( 'O/S Install',       'SATA busmaster bit not set' );
+    ktReason_Panic_BootManagerC000000F                 = ( 'Panic',             'Hardware Changed' );
+    ktReason_BootManager_Image_corrupt                 = ( 'Unknown',           'BOOTMGR Image corrupt' );
+    ktReason_Panic_MP_BIOS_IO_APIC                     = ( 'Panic',             'MP-BIOS/IO-APIC' );
+    ktReason_Panic_HugeMemory                          = ( 'Panic',             'Huge memory assertion' );
+    ktReason_Panic_IOAPICDoesntWork                    = ( 'Panic',             'IO-APIC and timer does not work' );
+    ktReason_Panic_TxUnitHang                          = ( 'Panic',             'Tx Unit Hang' );
+    ktReason_XPCOM_Exit_Minus_11                       = ( 'API / (XP)COM',     'exit -11' );
+    ktReason_XPCOM_VBoxSVC_Hang                        = ( 'API / (XP)COM',     'VBoxSVC hang' );
+    ktReason_XPCOM_VBoxSVC_Hang_Plus_Heap_Corruption   = ( 'API / (XP)COM',     'VBoxSVC hang + heap corruption' );
+    ktReason_XPCOM_NS_ERROR_CALL_FAILED                = ( 'API / (XP)COM',     'NS_ERROR_CALL_FAILED' );
+    ktReason_Unknown_Heap_Corruption                   = ( 'Unknown',           'Heap corruption' );
+    ktReason_Unknown_Reboot_Loop                       = ( 'Unknown',           'Reboot loop' );
+    ktReason_Unknown_File_Not_Found                    = ( 'Unknown',           'File not found' );
+    ktReason_Unknown_VM_Crash                          = ( 'Unknown',           'VM crash' );
+    ktReason_Unknown_HalReturnToFirmware               = ( 'Unknown',           'HalReturnToFirmware' );
+    ktReason_VMM_kvm_lock_spinning                     = ( 'VMM',               'kvm_lock_spinning' );
+    ktReason_Ignore_Buggy_Test_Driver                  = ( 'Ignore',            'Buggy test driver' );
+    ktReason_Ignore_Stale_Files                        = ( 'Ignore',            'Stale files' );
+    ktReason_Buggy_Build_Broken_Build                  = ( 'Broken Build',      'Buggy build' );
+    ktReason_Unknown_VM_Start_Error                    = ( 'Unknown',           'VM Start Error' );
+    ktReason_Unknown_VM_Runtime_Error                  = ( 'Unknown',           'VM Runtime Error' );
+    ktReason_GuestBug_CompizVBoxQt                     = ( 'Guest Bug',         'Compiz + VirtualBox Qt GUI crash' );
+    ## @}
+
+    ## BSOD category.
+    ksBsodCategory      = 'BSOD';
+    ## Special reason indicating that the flesh and blood sheriff has work to do.
+    ksBsodAddNew        = 'Add new BSOD';
+
+    ## Unit test category.
+    ksUnitTestCategory  = 'Unit';
+    ## Special reason indicating that the flesh and blood sheriff has work to do.
+    ksUnitTestAddNew    = 'Add new';
+
+    ## Used for indica that we shouldn't report anything for this test result ID and
+    ## consider promoting the previous error to test set level if it's the only one.
+    ktHarmless = ( 'Probably', 'Caused by previous error' );
+
+
+    def caseClosed(self, oCaseFile):
+        """
+        Reports the findings in the case and closes it.
+        """
+        #
+        # Log it and create a dReasonForReasultId we can use below.
+        #
+        dCommentForResultId = oCaseFile.dCommentForResultId;
+        if oCaseFile.dReasonForResultId:
+            # Must weed out ktHarmless.
+            dReasonForResultId = {};
+            for idKey, tReason in oCaseFile.dReasonForResultId.items():
+                if tReason is not self.ktHarmless:
+                    dReasonForResultId[idKey] = tReason;
+            if not dReasonForResultId:
+                self.vprint(u'TODO: Closing %s without a real reason, only %s.'
+                            % (oCaseFile.sName, oCaseFile.dReasonForResultId));
+                return False;
+
+            # Try promote to single reason.
+            atValues = dReasonForResultId.values();
+            fSingleReason = True;
+            if len(dReasonForResultId) == 1 and dReasonForResultId.keys()[0] != oCaseFile.oTestSet.idTestResult:
+                self.dprint(u'Promoting single reason to whole set: %s' % (atValues[0],));
+            elif len(dReasonForResultId) > 1 and len(atValues) == atValues.count(atValues[0]):
+                self.dprint(u'Merged %d reasons to a single one: %s' % (len(atValues), atValues[0]));
+            else:
+                fSingleReason = False;
+            if fSingleReason:
+                dReasonForResultId = { oCaseFile.oTestSet.idTestResult: atValues[0], };
+                if dCommentForResultId:
+                    dCommentForResultId = { oCaseFile.oTestSet.idTestResult: dCommentForResultId.values()[0], };
+        elif oCaseFile.tReason is not None:
+            dReasonForResultId = { oCaseFile.oTestSet.idTestResult: oCaseFile.tReason, };
+        else:
+            self.vprint(u'Closing %s without a reason - this should not happen!' % (oCaseFile.sName,));
+            return False;
+
+        self.vprint(u'Closing %s with following reason%s: %s'
+                    % ( oCaseFile.sName, 's' if dReasonForResultId > 0 else '', dReasonForResultId, ));
+
+        #
+        # Add the test failure reason record(s).
+        #
+        for idTestResult, tReason in dReasonForResultId.items():
+            oFailureReason = self.getFailureReason(tReason);
+            if oFailureReason is not None:
+                sComment = 'Set by $Revision: 127929 $' # Handy for reverting later.
+                if idTestResult in dCommentForResultId:
+                    sComment += ': ' + dCommentForResultId[idTestResult];
+
+                oAdd = TestResultFailureData();
+                oAdd.initFromValues(idTestResult    = idTestResult,
+                                    idFailureReason = oFailureReason.idFailureReason,
+                                    uidAuthor       = self.uidSelf,
+                                    idTestSet       = oCaseFile.oTestSet.idTestSet,
+                                    sComment        = sComment,);
+                if self.oConfig.fRealRun:
+                    try:
+                        self.oTestResultFailureLogic.addEntry(oAdd, self.uidSelf, fCommit = True);
+                    except Exception as oXcpt:
+                        self.eprint(u'caseClosed: Exception "%s" while adding reason %s for %s'
+                                    % (oXcpt, oAdd, oCaseFile.sLongName,));
+            else:
+                self.eprint(u'caseClosed: Cannot locate failure reason: %s / %s' % ( tReason[0], tReason[1],));
+        return True;
+
+    #
+    # Tools for assiting log parsing.
+    #
+
+    @staticmethod
+    def matchFollowedByLines(sStr, off, asFollowingLines):
+        """ Worker for isThisFollowedByTheseLines. """
+
+        # Advance off to the end of the line.
+        off = sStr.find('\n', off);
+        if off < 0:
+            return False;
+        off += 1;
+
+        # Match each string with the subsequent lines.
+        for iLine, sLine in enumerate(asFollowingLines):
+            offEnd = sStr.find('\n', off);
+            if offEnd < 0:
+                return  iLine + 1 == len(asFollowingLines) and sStr.find(sLine, off) < 0;
+            if sLine and sStr.find(sLine, off, offEnd) < 0:
+                return False;
+
+            # next line.
+            off = offEnd + 1;
+
+        return True;
+
+    @staticmethod
+    def isThisFollowedByTheseLines(sStr, sFirst, asFollowingLines):
+        """
+        Looks for a line contining sFirst which is then followed by lines
+        with the strings in asFollowingLines.  (No newline chars anywhere!)
+        Returns True / False.
+        """
+        off = sStr.find(sFirst, 0);
+        while off >= 0:
+            if VirtualTestSheriff.matchFollowedByLines(sStr, off, asFollowingLines):
+                return True;
+            off = sStr.find(sFirst, off + 1);
+        return False;
+
+    @staticmethod
+    def findAndReturnRestOfLine(sHaystack, sNeedle):
+        """
+        Looks for sNeedle in sHaystack.
+        Returns The text following the needle up to the end of the line.
+        Returns None if not found.
+        """
+        if sHaystack is None:
+            return None;
+        off = sHaystack.find(sNeedle);
+        if off < 0:
+            return None;
+        off += len(sNeedle)
+        offEol = sHaystack.find('\n', off);
+        if offEol < 0:
+            offEol = len(sHaystack);
+        return sHaystack[off:offEol]
+
+    @staticmethod
+    def findInAnyAndReturnRestOfLine(asHaystacks, sNeedle):
+        """
+        Looks for sNeedle in zeroe or more haystacks (asHaystack).
+        Returns The text following the first needed found up to the end of the line.
+        Returns None if not found.
+        """
+        for sHaystack in asHaystacks:
+            sRet = VirtualTestSheriff.findAndReturnRestOfLine(sHaystack, sNeedle);
+            if sRet is not None:
+                return sRet;
+        return None;
+
+
+    #
+    # The investigative units.
+    #
+
+    katSimpleInstallUninstallMainLogReasons = [
+        # ( Whether to stop on hit, reason tuple, needle text. )
+        ( False, ktReason_Host_LeftoverService,
+          'SERVICE_NAME: vbox' ),
+    ];
+
+    kdatSimpleInstallUninstallMainLogReasonsPerOs = {
+        'darwin': [
+            # ( Whether to stop on hit, reason tuple, needle text. )
+            ( True, ktReason_Host_DriverNotUnloading,
+              'Can\'t remove kext org.virtualbox.kext.VBoxDrv; services failed to terminate - 0xe00002c7' ),
+        ],
+        'linux': [
+            # ( Whether to stop on hit, reason tuple, needle text. )
+            ( True, ktReason_Host_DriverNotCompilable,
+              'This system is not currently set up to build kernel modules' ),
+            ( True, ktReason_Host_DriverNotCompilable,
+              'This system is currently not set up to build kernel modules' ),
+            ( True, ktReason_Host_InstallationFailed,
+              'vboxdrv.sh: failed: Look at /var/log/vbox-install.log to find out what went wrong.' ),
+            ( True, ktReason_Host_DriverNotUnloading,
+              'Cannot unload module vboxdrv'),
+        ],
+    };
+
+
+    def investigateInstallUninstallFailure(self, oCaseFile, oFailedResult, sResultLog, fInstall):
+        """
+        Investigates an install or uninstall failure.
+
+        We lump the two together since the installation typically also performs
+        an uninstall first and will be seeing similar issues to the uninstall.
+        """
+
+        if fInstall and oFailedResult.enmStatus == TestSetData.ksTestStatus_TimedOut:
+            oCaseFile.noteReasonForId(self.ktReason_Host_Install_Hang, oFailedResult.idTestResult)
+            return True;
+
+        atSimple = self.katSimpleInstallUninstallMainLogReasons;
+        if oCaseFile.oTestBox.sOs in self.kdatSimpleInstallUninstallMainLogReasonsPerOs:
+            atSimple = self.kdatSimpleInstallUninstallMainLogReasonsPerOs[oCaseFile.oTestBox.sOs] + atSimple;
+
+        fFoundSomething = False;
+        for fStopOnHit, tReason, sNeedle in atSimple:
+            if sResultLog.find(sNeedle) > 0:
+                oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
+                if fStopOnHit:
+                    return True;
+                fFoundSomething = True;
+
+        return fFoundSomething if fFoundSomething else None;
+
+
+    def investigateBadTestBox(self, oCaseFile):
+        """
+        Checks out bad-testbox statuses.
+        """
+        _ = oCaseFile;
+        return False;
+
+
+    def investigateVBoxUnitTest(self, oCaseFile):
+        """
+        Checks out a VBox unittest problem.
+        """
+
+        #
+        # Process simple test case failures first, using their name as reason.
+        # We do the reason management just like for BSODs.
+        #
+        cRelevantOnes   = 0;
+        sMainLog        = oCaseFile.getMainLog();
+        aoFailedResults = oCaseFile.oTree.getListOfFailures();
+        for oFailedResult in aoFailedResults:
+            if oFailedResult is oCaseFile.oTree:
+                self.vprint('TODO: toplevel failure');
+                cRelevantOnes += 1
+
+            elif oFailedResult.sName == 'Installing VirtualBox':
+                sResultLog = TestSetData.extractLogSectionElapsed(sMainLog, oFailedResult.tsCreated, oFailedResult.tsElapsed);
+                self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = True)
+                cRelevantOnes += 1
+
+            elif oFailedResult.sName == 'Uninstalling VirtualBox':
+                sResultLog = TestSetData.extractLogSectionElapsed(sMainLog, oFailedResult.tsCreated, oFailedResult.tsElapsed);
+                self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = False)
+                cRelevantOnes += 1
+
+            elif oFailedResult.oParent is not None:
+                # Get the 2nd level node because that's where we'll find the unit test name.
+                while oFailedResult.oParent.oParent is not None:
+                    oFailedResult = oFailedResult.oParent;
+
+                # Only report a failure once.
+                if oFailedResult.idTestResult not in oCaseFile.dReasonForResultId:
+                    sKey = oFailedResult.sName;
+                    if sKey.startswith('testcase/'):
+                        sKey = sKey[9:];
+                    if sKey in self.asUnitTestReasons:
+                        tReason = ( self.ksUnitTestCategory, sKey );
+                        oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
+                    else:
+                        self.dprint(u'Unit test failure "%s" not found in %s;' % (sKey, self.asUnitTestReasons));
+                        tReason = ( self.ksUnitTestCategory, self.ksUnitTestAddNew );
+                        oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult, sComment = sKey);
+                    cRelevantOnes += 1
+            else:
+                self.vprint(u'Internal error: expected oParent to NOT be None for %s' % (oFailedResult,));
+
+        #
+        # If we've caught all the relevant ones by now, report the result.
+        #
+        if len(oCaseFile.dReasonForResultId) >= cRelevantOnes:
+            return self.caseClosed(oCaseFile);
+        return False;
+
+    def extractGuestCpuStack(self, sInfoText):
+        """
+        Extracts the guest CPU stacks from the input file.
+
+        Returns a dictionary keyed by the CPU number, value being a list of
+        raw stack lines (no header).
+        Returns empty dictionary if no stacks where found.
+        """
+        dRet = {};
+        off = 0;
+        while True:
+            # Find the stack.
+            offStart = sInfoText.find('=== start guest stack VCPU ', off);
+            if offStart < 0:
+                break;
+            offEnd  = sInfoText.find('=== end guest stack', offStart + 20);
+            if offEnd >= 0:
+                offEnd += 3;
+            else:
+                offEnd = sInfoText.find('=== start guest stack VCPU', offStart + 20);
+                if offEnd < 0:
+                    offEnd = len(sInfoText);
+
+            sStack = sInfoText[offStart : offEnd];
+            sStack = sStack.replace('\r',''); # paranoia
+            asLines = sStack.split('\n');
+
+            # Figure the CPU.
+            asWords = asLines[0].split();
+            if len(asWords) < 6 or not asWords[5].isdigit():
+                break;
+            iCpu = int(asWords[5]);
+
+            # Add it and advance.
+            dRet[iCpu] = [sLine.rstrip() for sLine in asLines[2:-1]]
+            off = offEnd;
+        return dRet;
+
+    def investigateInfoKvmLockSpinning(self, oCaseFile, sInfoText, dLogs):
+        """ Investigates kvm_lock_spinning deadlocks """
+        #
+        # Extract the stacks.  We need more than one CPU to create a deadlock.
+        #
+        dStacks = self.extractGuestCpuStack(sInfoText);
+        self.dprint('kvm_lock_spinning: found %s stacks' % (len(dStacks),));
+        if len(dStacks) >= 2:
+            #
+            # Examin each of the stacks.  Each must have kvm_lock_spinning in
+            # one of the first three entries.
+            #
+            cHits = 0;
+            for iCpu in dStacks:
+                asBacktrace = dStacks[iCpu];
+                for iFrame in xrange(min(3, len(asBacktrace))):
+                    if asBacktrace[iFrame].find('kvm_lock_spinning') >= 0:
+                        cHits += 1;
+                        break;
+            self.dprint('kvm_lock_spinning: %s/%s hits' % (cHits, len(dStacks),));
+            if cHits == len(dStacks):
+                return (True, self.ktReason_VMM_kvm_lock_spinning);
+
+        _ = dLogs; _ = oCaseFile;
+        return (False, None);
+
+    def investigateInfoHalReturnToFirmware(self, oCaseFile, sInfoText, dLogs):
+        """ Investigates HalReturnToFirmware hangs """
+        del oCaseFile
+        del sInfoText
+        del dLogs
+        # hope that's sufficient
+        return (True, self.ktReason_Unknown_HalReturnToFirmware);
+
+    ## Things we search a main or VM log for to figure out why something went bust.
+    katSimpleMainAndVmLogReasons = [
+        # ( Whether to stop on hit, reason tuple, needle text. )
+        ( False, ktReason_Guru_Generic,                             'GuruMeditation' ),
+        ( False, ktReason_Guru_Generic,                             'Guru Meditation' ),
+        ( True,  ktReason_Guru_VERR_IEM_INSTR_NOT_IMPLEMENTED,      'VERR_IEM_INSTR_NOT_IMPLEMENTED' ),
+        ( True,  ktReason_Guru_VERR_IEM_ASPECT_NOT_IMPLEMENTED,     'VERR_IEM_ASPECT_NOT_IMPLEMENTED' ),
+        ( True,  ktReason_Guru_VERR_TRPM_DONT_PANIC,                'VERR_TRPM_DONT_PANIC' ),
+        ( True,  ktReason_Guru_VERR_PGM_PHYS_PAGE_RESERVED,         'VERR_PGM_PHYS_PAGE_RESERVED' ),
+        ( True,  ktReason_Guru_VERR_VMX_INVALID_GUEST_STATE,        'VERR_VMX_INVALID_GUEST_STATE' ),
+        ( True,  ktReason_Guru_VINF_EM_TRIPLE_FAULT,                'VINF_EM_TRIPLE_FAULT' ),
+        ( True,  ktReason_Networking_Nonexistent_host_nic,
+          'rc=E_FAIL text="Nonexistent host networking interface, name \'eth0\' (VERR_INTERNAL_ERROR)"' ),
+        ( True,  ktReason_Host_Reboot_OSX_Watchdog_Timeout,         ': "OSX Watchdog Timeout: ' ),
+        ( False, ktReason_XPCOM_NS_ERROR_CALL_FAILED,
+          'Exception: 0x800706be (Call to remote object failed (NS_ERROR_CALL_FAILED))' ),
+        ( True,  ktReason_Host_HostMemoryLow,                       'HostMemoryLow' ),
+        ( True,  ktReason_Host_HostMemoryLow,                       'Failed to procure handy pages; rc=VERR_NO_MEMORY' ),
+        ( True,  ktReason_Unknown_File_Not_Found,
+          'Error: failed to start machine. Error message: File not found. (VERR_FILE_NOT_FOUND)' ),
+        ( True,  ktReason_Unknown_File_Not_Found, # lump it in with file-not-found for now.
+          'Error: failed to start machine. Error message: Not supported. (VERR_NOT_SUPPORTED)' ),
+        ( False, ktReason_Unknown_VM_Crash,                         'txsDoConnectViaTcp: Machine state: Aborted' ),
+        ( True,  ktReason_Host_Modprobe_Failed,                     'Kernel driver not installed' ),
+        ( True,  ktReason_OSInstall_Sata_no_BM,                     'PCHS=14128/14134/8224' ),
+        ( True,  ktReason_Host_DoubleFreeHeap,                      'double free or corruption' ),
+        ( False, ktReason_Unknown_VM_Start_Error,                   'VMSetError: ' ),
+        ( False, ktReason_Unknown_VM_Runtime_Error,                 'Console: VM runtime error: fatal=true' ),
+    ];
+
+    ## Things we search a VBoxHardening.log file for to figure out why something went bust.
+    katSimpleVBoxHardeningLogReasons = [
+        # ( Whether to stop on hit, reason tuple, needle text. )
+        ( True,  ktReason_Host_DriverNotLoaded,                     'Error opening VBoxDrvStub:  STATUS_OBJECT_NAME_NOT_FOUND' ),
+        ( True,  ktReason_Host_NotSignedWithBuildCert,              'Not signed with the build certificate' ),
+    ];
+
+    ## Things we search a kernel.log file for to figure out why something went bust.
+    katSimpleKernelLogReasons = [
+        # (  Whether to stop on hit, reason tuple, needle text. )
+        ( True,  ktReason_Panic_HugeMemory,                         'mm/huge_memory.c:1988' ),
+        ( True,  ktReason_Panic_IOAPICDoesntWork,                   'IO-APIC + timer doesn''t work' ),
+        ( True,  ktReason_Panic_TxUnitHang,                         'Detected Tx Unit Hang' ),
+        ( True,  ktReason_GuestBug_CompizVBoxQt,                    'error 4 in libQt5CoreVBox' ),
+        ( True,  ktReason_GuestBug_CompizVBoxQt,                    'error 4 in libgtk-3' ),
+    ];
+
+    ## Things we search the _RIGHT_ _STRIPPED_ vgatext for.
+    katSimpleVgaTextReasons = [
+        # ( Whether to stop on hit, reason tuple, needle text. )
+        ( True,  ktReason_Panic_MP_BIOS_IO_APIC,
+          "..MP-BIOS bug: 8254 timer not connected to IO-APIC\n\n" ),
+        ( True,  ktReason_Panic_MP_BIOS_IO_APIC,
+          "..MP-BIOS bug: 8254 timer not connected to IO-APIC\n"
+          "...trying to set up timer (IRQ0) through the 8259A ...  failed.\n"
+          "...trying to set up timer as Virtual Wire IRQ... failed.\n"
+          "...trying to set up timer as ExtINT IRQ... failed :(.\n"
+          "Kernel panic - not syncing: IO-APIC + timer doesn't work!  Boot with apic=debug\n"
+          "and send a report.  Then try booting with the 'noapic' option\n"
+          "\n" ),
+        ( True,  ktReason_OSInstall_GRUB_hang,
+          "-----\nGRUB Loading stage2..\n\n\n\n" ),
+        ( True,  ktReason_OSInstall_GRUB_hang,
+          "-----\nGRUB Loading stage2...\n\n\n\n" ), # the 3 dot hang appears to be less frequent
+        ( True,  ktReason_OSInstall_GRUB_hang,
+          "-----\nGRUB Loading stage2....\n\n\n\n" ), # the 4 dot hang appears to be very infrequent
+        ( True,  ktReason_OSInstall_GRUB_hang,
+          "-----\nGRUB Loading stage2.....\n\n\n\n" ), # the 5 dot hang appears to be more frequent again
+        ( True,  ktReason_OSInstall_Udev_hang,
+          "\nStarting udev:\n\n\n\n" ),
+        ( True,  ktReason_OSInstall_Udev_hang,
+          "\nStarting udev:\n------" ),
+        ( True,  ktReason_Panic_BootManagerC000000F,
+          "Windows failed to start. A recent hardware or software change might be the" ),
+        ( True,  ktReason_BootManager_Image_corrupt,
+          "BOOTMGR image is corrupt.  The system cannot boot." ),
+    ];
+
+    ## Things we search for in the info.txt file.  Require handlers for now.
+    katInfoTextHandlers = [
+        # ( Trigger text,                       handler method )
+        ( "kvm_lock_spinning",                  investigateInfoKvmLockSpinning ),
+        ( "HalReturnToFirmware",                investigateInfoHalReturnToFirmware ),
+    ];
+
+    ## Mapping screenshot/failure SHA-256 hashes to failure reasons.
+    katSimpleScreenshotHashReasons = [
+        # ( Whether to stop on hit, reason tuple, lowercased sha-256 of PIL.Image.tostring output )
+        ( True,  ktReason_BSOD_Recovery,                    '576f8e38d62b311cac7e3dc3436a0d0b9bd8cfd7fa9c43aafa95631520a45eac' ),
+        ( True,  ktReason_BSOD_Automatic_Repair,            'c6a72076cc619937a7a39cfe9915b36d94cee0d4e3ce5ce061485792dcee2749' ),
+        ( True,  ktReason_BSOD_Automatic_Repair,            '26c4d8a724ff2c5e1051f3d5b650dbda7b5fdee0aa3e3c6059797f7484a515df' ),
+        ( True,  ktReason_BSOD_0000007F,                    '57e1880619e13042a87100e7a38c8974b85ce3866501be621bea0cc696bb2c63' ),
+        ( True,  ktReason_BSOD_000000D1,                    '134621281f00a3f8aeeb7660064bffbf6187ed56d5852142328d0bcb18ef0ede' ),
+        ( True,  ktReason_BSOD_000000D1,                    '279f11258150c9d2fef041eca65501f3141da8df39256d8f6377e897e3b45a93' ),
+        ( True,  ktReason_BSOD_C0000225,                    'bd13a144be9dcdfb16bc863ff4c8f02a86e263c174f2cd5ffd27ca5f3aa31789' ),
+        ( True,  ktReason_BSOD_C0000225,                    '8348b465e7ee9e59dd4e785880c57fd8677de05d11ac21e786bfde935307b42f' ),
+        ( True,  ktReason_BSOD_C0000225,                    '1316e1fc818a73348412788e6910b8c016f237d8b4e15b20caf4a866f7a7840e' ),
+        ( True,  ktReason_BSOD_C0000225,                    '54e0acbff365ce20a85abbe42bcd53647b8b9e80c68e45b2cd30e86bf177a0b5' ),
+        ( True,  ktReason_BSOD_C0000225,                    '50fec50b5199923fa48b3f3e782687cc381e1c8a788ebda14e6a355fbe3bb1b3' ),
+    ];
+
+    def investigateVMResult(self, oCaseFile, oFailedResult, sResultLog):
+        """
+        Investigates a failed VM run.
+        """
+
+        def investigateLogSet():
+            """
+            Investigates the current set of VM related logs.
+            """
+            self.dprint('investigateLogSet: lengths: result log %u, VM log %u, kernel log %u, vga text %u, info text %u'
+                        % ( len(sResultLog if sResultLog else ''),
+                            len(sVMLog     if sVMLog else ''),
+                            len(sKrnlLog   if sKrnlLog else ''),
+                            len(sVgaText   if sVgaText else ''),
+                            len(sInfoText  if sInfoText else ''), ));
+
+            #self.dprint(u'main.log<<<\n%s\n<<<\n' % (sResultLog,));
+            #self.dprint(u'vbox.log<<<\n%s\n<<<\n' % (sVMLog,));
+            #self.dprint(u'krnl.log<<<\n%s\n<<<\n' % (sKrnlLog,));
+            #self.dprint(u'vgatext.txt<<<\n%s\n<<<\n' % (sVgaText,));
+            #self.dprint(u'info.txt<<<\n%s\n<<<\n' % (sInfoText,));
+
+            # TODO: more
+
+            #
+            # Look for BSODs. Some stupid stupid inconsistencies in reason and log messages here, so don't try prettify this.
+            #
+            sDetails = self.findInAnyAndReturnRestOfLine([ sVMLog, sResultLog ],
+                                                         'GIM: HyperV: Guest indicates a fatal condition! P0=');
+            if sDetails is not None:
+                # P0=%#RX64 P1=%#RX64 P2=%#RX64 P3=%#RX64 P4=%#RX64 "
+                sKey = sDetails.split(' ', 1)[0];
+                try:    sKey = '0x%08X' % (int(sKey, 16),);
+                except: pass;
+                if sKey in self.asBsodReasons:
+                    tReason = ( self.ksBsodCategory, sKey );
+                elif sKey.lower() in self.asBsodReasons: # just in case.
+                    tReason = ( self.ksBsodCategory, sKey.lower() );
+                else:
+                    self.dprint(u'BSOD "%s" not found in %s;' % (sKey, self.asBsodReasons));
+                    tReason = ( self.ksBsodCategory, self.ksBsodAddNew );
+                return oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult, sComment = sDetails.strip());
+
+            #
+            # Look for linux panic.
+            #
+            if sKrnlLog is not None:
+                for fStopOnHit, tReason, sNeedle in self.katSimpleKernelLogReasons:
+                    if sKrnlLog.find(sNeedle) > 0:
+                        oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
+                        if fStopOnHit:
+                            return True;
+                        fFoundSomething = True;
+
+            #
+            # Loop thru the simple stuff.
+            #
+            fFoundSomething = False;
+            for fStopOnHit, tReason, sNeedle in self.katSimpleMainAndVmLogReasons:
+                if sResultLog.find(sNeedle) > 0 or (sVMLog is not None and sVMLog.find(sNeedle) > 0):
+                    oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
+                    if fStopOnHit:
+                        return True;
+                    fFoundSomething = True;
+
+            # Continue with vga text.
+            if sVgaText:
+                for fStopOnHit, tReason, sNeedle in self.katSimpleVgaTextReasons:
+                    if sVgaText.find(sNeedle) > 0:
+                        oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
+                        if fStopOnHit:
+                            return True;
+                        fFoundSomething = True;
+            _ = sInfoText;
+
+            # Continue with screen hashes.
+            if sScreenHash is not None:
+                for fStopOnHit, tReason, sHash in self.katSimpleScreenshotHashReasons:
+                    if sScreenHash == sHash:
+                        oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
+                        if fStopOnHit:
+                            return True;
+                        fFoundSomething = True;
+
+            # Check VBoxHardening.log.
+            if sNtHardLog is not None:
+                for fStopOnHit, tReason, sNeedle in self.katSimpleVBoxHardeningLogReasons:
+                    if sNtHardLog.find(sNeedle) > 0:
+                        oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
+                        if fStopOnHit:
+                            return True;
+                        fFoundSomething = True;
+
+            #
+            # Complicated stuff.
+            #
+            dLogs = {
+                'sVMLog':       sVMLog,
+                'sNtHardLog':   sNtHardLog,
+                'sScreenHash':  sScreenHash,
+                'sKrnlLog':     sKrnlLog,
+                'sVgaText':     sVgaText,
+                'sInfoText':    sInfoText,
+            };
+
+            # info.txt.
+            if sInfoText:
+                for sNeedle, fnHandler in self.katInfoTextHandlers:
+                    if sInfoText.find(sNeedle) > 0:
+                        (fStop, tReason) = fnHandler(self, oCaseFile, sInfoText, dLogs);
+                        if tReason is not None:
+                            oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
+                            if fStop:
+                                return True;
+                            fFoundSomething = True;
+
+            #
+            # Check for repeated reboots...
+            #
+            if sVMLog is not None:
+                cResets = sVMLog.count('Changing the VM state from \'RUNNING\' to \'RESETTING\'');
+                if cResets > 10:
+                    return oCaseFile.noteReasonForId(self.ktReason_Unknown_Reboot_Loop, oFailedResult.idTestResult,
+                                                     sComment = 'Counted %s reboots' % (cResets,));
+
+            return fFoundSomething;
+
+        #
+        # Check if we got any VM or/and kernel logs.  Treat them as sets in
+        # case we run multiple VMs here (this is of course ASSUMING they
+        # appear in the order that terminateVmBySession uploads them).
+        #
+        cTimes      = 0;
+        sVMLog      = None;
+        sNtHardLog  = None;
+        sScreenHash = None;
+        sKrnlLog    = None;
+        sVgaText    = None;
+        sInfoText   = None;
+        for oFile in oFailedResult.aoFiles:
+            if oFile.sKind == TestResultFileData.ksKind_LogReleaseVm:
+                if 'VBoxHardening.log' not in oFile.sFile:
+                    if sVMLog is not None:
+                        if investigateLogSet() is True:
+                            return True;
+                        cTimes += 1;
+                    sInfoText   = None;
+                    sVgaText    = None;
+                    sKrnlLog    = None;
+                    sScreenHash = None;
+                    sNtHardLog  = None;
+                    sVMLog      = oCaseFile.getLogFile(oFile);
+                else:
+                    sNtHardLog  = oCaseFile.getLogFile(oFile);
+            elif oFile.sKind == TestResultFileData.ksKind_LogGuestKernel:
+                sKrnlLog  = oCaseFile.getLogFile(oFile);
+            elif oFile.sKind == TestResultFileData.ksKind_InfoVgaText:
+                sVgaText  = '\n'.join([sLine.rstrip() for sLine in oCaseFile.getLogFile(oFile).split('\n')]);
+            elif oFile.sKind == TestResultFileData.ksKind_InfoCollection:
+                sInfoText = oCaseFile.getLogFile(oFile);
+            elif oFile.sKind == TestResultFileData.ksKind_ScreenshotFailure:
+                sScreenHash = oCaseFile.getScreenshotSha256(oFile);
+                if sScreenHash is not None:
+                    sScreenHash = sScreenHash.lower();
+                    self.vprint(u'%s  %s' % ( sScreenHash, oFile.sFile,));
+
+        if    (   sVMLog     is not None \
+               or sNtHardLog is not None \
+               or cTimes == 0) \
+          and investigateLogSet() is True:
+            return True;
+
+        return None;
+
+
+    def isResultFromVMRun(self, oFailedResult, sResultLog):
+        """
+        Checks if this result and corresponding log snippet looks like a VM run.
+        """
+
+        # Look for startVmEx/ startVmAndConnectToTxsViaTcp and similar output in the log.
+        if sResultLog.find(' startVm') > 0:
+            return True;
+
+        # Any other indicators? No?
+        _ = oFailedResult;
+        return False;
+
+    def investigateVBoxVMTest(self, oCaseFile, fSingleVM):
+        """
+        Checks out a VBox VM test.
+
+        This is generic investigation of a test running one or more VMs, like
+        for example a smoke test or a guest installation test.
+
+        The fSingleVM parameter is a hint, which probably won't come in useful.
+        """
+        _ = fSingleVM;
+
+        #
+        # Get a list of test result failures we should be looking into and the main log.
+        #
+        aoFailedResults = oCaseFile.oTree.getListOfFailures();
+        sMainLog        = oCaseFile.getMainLog();
+
+        #
+        # There are a set of errors ending up on the top level result record.
+        # Should deal with these first.
+        #
+        if len(aoFailedResults) == 1 and aoFailedResults[0] == oCaseFile.oTree:
+            # Check if we've just got that XPCOM client smoke test shutdown issue.  This will currently always
+            # be reported on the top result because vboxinstall.py doesn't add an error for it.  It is easy to
+            # ignore other failures in the test if we're not a little bit careful here.
+            if sMainLog.find('vboxinstaller: Exit code: -11 (') > 0:
+                oCaseFile.noteReason(self.ktReason_XPCOM_Exit_Minus_11);
+                return self.caseClosed(oCaseFile);
+
+            # Hang after starting VBoxSVC (e.g. idTestSet=136307258)
+            if self.isThisFollowedByTheseLines(sMainLog, 'oVBoxMgr=<vboxapi.VirtualBoxManager object at',
+                                               (' Timeout: ', ' Attempting to abort child...',) ):
+                if sMainLog.find('*** glibc detected *** /') > 0:
+                    oCaseFile.noteReason(self.ktReason_XPCOM_VBoxSVC_Hang_Plus_Heap_Corruption);
+                else:
+                    oCaseFile.noteReason(self.ktReason_XPCOM_VBoxSVC_Hang);
+                return self.caseClosed(oCaseFile);
+
+            # Look for heap corruption without visible hang.
+            if   sMainLog.find('*** glibc detected *** /') > 0 \
+              or sMainLog.find("-1073740940") > 0: # STATUS_HEAP_CORRUPTION / 0xc0000374
+                oCaseFile.noteReason(self.ktReason_Unknown_Heap_Corruption);
+                return self.caseClosed(oCaseFile);
+
+            # Out of memory w/ timeout.
+            if sMainLog.find('sErrId=HostMemoryLow') > 0:
+                oCaseFile.noteReason(self.ktReason_Host_HostMemoryLow);
+                return self.caseClosed(oCaseFile);
+
+            # Stale files like vts_rm.exe (windows).
+            offEnd = sMainLog.rfind('*** The test driver exits successfully. ***');
+            if offEnd > 0 and sMainLog.find('[Error 145] The directory is not empty: ', offEnd) > 0:
+                oCaseFile.noteReason(self.ktReason_Ignore_Stale_Files);
+                return self.caseClosed(oCaseFile);
+
+        #
+        # XPCOM screwup
+        #
+        if   sMainLog.find('AttributeError: \'NoneType\' object has no attribute \'addObserver\'') > 0:
+            oCaseFile.noteReason(self.ktReason_Buggy_Build_Broken_Build);
+            return self.caseClosed(oCaseFile);
+
+        #
+        # Go thru each failed result.
+        #
+        for oFailedResult in aoFailedResults:
+            self.dprint(u'Looking at test result #%u - %s' % (oFailedResult.idTestResult, oFailedResult.getFullName(),));
+            sResultLog = TestSetData.extractLogSectionElapsed(sMainLog, oFailedResult.tsCreated, oFailedResult.tsElapsed);
+            if oFailedResult.sName == 'Installing VirtualBox':
+                self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = True)
+
+            elif oFailedResult.sName == 'Uninstalling VirtualBox':
+                self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = False)
+
+            elif self.isResultFromVMRun(oFailedResult, sResultLog):
+                self.investigateVMResult(oCaseFile, oFailedResult, sResultLog);
+
+            elif sResultLog.find('most likely not unique') > 0:
+                oCaseFile.noteReasonForId(self.ktReason_Host_NetworkMisconfiguration, oFailedResult.idTestResult)
+            elif sResultLog.find('Exception: 0x800706be (Call to remote object failed (NS_ERROR_CALL_FAILED))') > 0:
+                oCaseFile.noteReasonForId(self.ktReason_XPCOM_NS_ERROR_CALL_FAILED, oFailedResult.idTestResult);
+
+            elif sResultLog.find('The machine is not mutable (state is ') > 0:
+                self.vprint('Ignoring "machine not mutable" error as it is probably due to an earlier problem');
+                oCaseFile.noteReasonForId(self.ktHarmless, oFailedResult.idTestResult);
+
+            elif  sResultLog.find('** error: no action was specified') > 0 \
+               or sResultLog.find('(len(self._asXml, asText))') > 0:
+                oCaseFile.noteReasonForId(self.ktReason_Ignore_Buggy_Test_Driver, oFailedResult.idTestResult);
+
+            else:
+                self.vprint(u'TODO: Cannot place idTestResult=%u - %s' % (oFailedResult.idTestResult, oFailedResult.sName,));
+                self.dprint(u'%s + %s <<\n%s\n<<' % (oFailedResult.tsCreated, oFailedResult.tsElapsed, sResultLog,));
+
+        #
+        # Report home and close the case if we got them all, otherwise log it.
+        #
+        if len(oCaseFile.dReasonForResultId) >= len(aoFailedResults):
+            return self.caseClosed(oCaseFile);
+
+        if oCaseFile.dReasonForResultId:
+            self.vprint(u'TODO: Got %u out of %u - close, but no cigar. :-/'
+                        % (len(oCaseFile.dReasonForResultId), len(aoFailedResults)));
+        else:
+            self.vprint(u'XXX: Could not figure out anything at all! :-(');
+        return False;
+
+
+    def reasoningFailures(self):
+        """
+        Guess the reason for failures.
+        """
+        #
+        # Get a list of failed test sets without any assigned failure reason.
+        #
+        cGot = 0;
+        aoTestSets = self.oTestSetLogic.fetchFailedSetsWithoutReason(cHoursBack = self.oConfig.cHoursBack, tsNow = self.tsNow);
+        for oTestSet in aoTestSets:
+            self.dprint(u'');
+            self.dprint(u'reasoningFailures: Checking out test set #%u, status %s'  % ( oTestSet.idTestSet, oTestSet.enmStatus,))
+
+            #
+            # Open a case file and assign it to the right investigator.
+            #
+            (oTree, _ ) = self.oTestResultLogic.fetchResultTree(oTestSet.idTestSet);
+            oBuild      = BuildDataEx().initFromDbWithId(       self.oDb, oTestSet.idBuild,       oTestSet.tsCreated);
+            oTestBox    = TestBoxData().initFromDbWithGenId(    self.oDb, oTestSet.idGenTestBox);
+            oTestGroup  = TestGroupData().initFromDbWithId(     self.oDb, oTestSet.idTestGroup,   oTestSet.tsCreated);
+            oTestCase   = TestCaseDataEx().initFromDbWithGenId( self.oDb, oTestSet.idGenTestCase, oTestSet.tsConfig);
+
+            oCaseFile = VirtualTestSheriffCaseFile(self, oTestSet, oTree, oBuild, oTestBox, oTestGroup, oTestCase);
+
+            if oTestSet.enmStatus == TestSetData.ksTestStatus_BadTestBox:
+                self.dprint(u'investigateBadTestBox is taking over %s.' % (oCaseFile.sLongName,));
+                fRc = self.investigateBadTestBox(oCaseFile);
+
+            elif oCaseFile.isVBoxUnitTest():
+                self.dprint(u'investigateVBoxUnitTest is taking over %s.' % (oCaseFile.sLongName,));
+                fRc = self.investigateVBoxUnitTest(oCaseFile);
+
+            elif oCaseFile.isVBoxInstallTest():
+                self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
+                fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True);
+
+            elif oCaseFile.isVBoxUSBTest():
+                self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
+                fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True);
+
+            elif oCaseFile.isVBoxStorageTest():
+                self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
+                fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True);
+
+            elif oCaseFile.isVBoxGAsTest():
+                self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
+                fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True);
+
+            elif oCaseFile.isVBoxAPITest():
+                self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
+                fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True);
+
+            elif oCaseFile.isVBoxBenchmarkTest():
+                self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
+                fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = False);
+
+            elif oCaseFile.isVBoxSmokeTest():
+                self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
+                fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = False);
+
+            else:
+                self.vprint(u'reasoningFailures: Unable to classify test set: %s' % (oCaseFile.sLongName,));
+                fRc = False;
+            cGot += fRc is True;
+
+        self.vprint(u'reasoningFailures: Got %u out of %u' % (cGot, len(aoTestSets), ));
+        return 0;
+
+
+    def main(self):
+        """
+        The 'main' function.
+        Return exit code (0, 1, etc).
+        """
+        # Database stuff.
+        self.oDb                     = TMDatabaseConnection()
+        self.oTestResultLogic        = TestResultLogic(self.oDb);
+        self.oTestSetLogic           = TestSetLogic(self.oDb);
+        self.oFailureReasonLogic     = FailureReasonLogic(self.oDb);
+        self.oTestResultFailureLogic = TestResultFailureLogic(self.oDb);
+        self.asBsodReasons           = self.oFailureReasonLogic.fetchForSheriffByNamedCategory(self.ksBsodCategory);
+        self.asUnitTestReasons       = self.oFailureReasonLogic.fetchForSheriffByNamedCategory(self.ksUnitTestCategory);
+
+        # Get a fix on our 'now' before we do anything..
+        self.oDb.execute('SELECT CURRENT_TIMESTAMP - interval \'%s hours\'', (self.oConfig.cStartHoursAgo,));
+        self.tsNow = self.oDb.fetchOne();
+
+        # If we're suppost to commit anything we need to get our user ID.
+        rcExit = 0;
+        if self.oConfig.fRealRun:
+            self.oLogin = UserAccountLogic(self.oDb).tryFetchAccountByLoginName(VirtualTestSheriff.ksLoginName);
+            if self.oLogin is None:
+                rcExit = self.eprint('Cannot find my user account "%s"!' % (VirtualTestSheriff.ksLoginName,));
+            else:
+                self.uidSelf = self.oLogin.uid;
+
+        #
+        # Do the stuff.
+        #
+        if rcExit == 0:
+            rcExit  = self.selfCheck();
+        if rcExit == 0:
+            rcExit  = self.badTestBoxManagement();
+            rcExit2 = self.reasoningFailures();
+            if rcExit == 0:
+                rcExit = rcExit2;
+            # Redo the bad testbox management after failure reasons have been assigned (got timing issues).
+            if rcExit == 0:
+                rcExit = self.badTestBoxManagement();
+
+        # Cleanup.
+        self.oFailureReasonLogic     = None;
+        self.oTestResultFailureLogic = None;
+        self.oTestSetLogic           = None;
+        self.oTestResultLogic        = None;
+        self.oDb.close();
+        self.oDb = None;
+        if self.oLogFile is not None:
+            self.oLogFile.close();
+            self.oLogFile = None;
+        return rcExit;
+
+if __name__ == '__main__':
+    sys.exit(VirtualTestSheriff().main());
+