summaryrefslogtreecommitdiffstats
path: root/src/VBox/ValidationKit/testmanager/batch/filearchiver.py
diff options
context:
space:
mode:
Diffstat (limited to '')
-rwxr-xr-xsrc/VBox/ValidationKit/testmanager/batch/filearchiver.py275
1 files changed, 275 insertions, 0 deletions
diff --git a/src/VBox/ValidationKit/testmanager/batch/filearchiver.py b/src/VBox/ValidationKit/testmanager/batch/filearchiver.py
new file mode 100755
index 00000000..9c6bb8e8
--- /dev/null
+++ b/src/VBox/ValidationKit/testmanager/batch/filearchiver.py
@@ -0,0 +1,275 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# $Id: filearchiver.py $
+# pylint: disable=C0301
+
+"""
+A cronjob that compresses logs and other files, moving them to the
+g_ksZipFileAreaRootDir storage area.
+"""
+
+from __future__ import print_function;
+
+__copyright__ = \
+"""
+Copyright (C) 2012-2019 Oracle Corporation
+
+This file is part of VirtualBox Open Source Edition (OSE), as
+available from http://www.virtualbox.org. This file is free software;
+you can redistribute it and/or modify it under the terms of the GNU
+General Public License (GPL) as published by the Free Software
+Foundation, in version 2 as it comes in the "COPYING" file of the
+VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+
+The contents of this file may alternatively be used under the terms
+of the Common Development and Distribution License Version 1.0
+(CDDL) only, as it comes in the "COPYING.CDDL" file of the
+VirtualBox OSE distribution, in which case the provisions of the
+CDDL are applicable instead of those of the GPL.
+
+You may elect to license modified versions of this file under the
+terms and conditions of either the GPL or the CDDL or both.
+"""
+__version__ = "$Revision: 127855 $"
+
+# Standard python imports
+import sys
+import os
+from optparse import OptionParser; # pylint: disable=deprecated-module
+import time;
+import zipfile;
+
+# Add Test Manager's modules path
+g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.append(g_ksTestManagerDir)
+
+# Test Manager imports
+from common import utils;
+from testmanager import config;
+from testmanager.core.db import TMDatabaseConnection;
+from testmanager.core.testset import TestSetData, TestSetLogic;
+
+
+
+class FileArchiverBatchJob(object): # pylint: disable=R0903
+ """
+ Log+files comp
+ """
+
+ def __init__(self, oOptions):
+ """
+ Parse command line
+ """
+ self.fVerbose = oOptions.fVerbose;
+ self.sSrcDir = config.g_ksFileAreaRootDir;
+ self.sDstDir = config.g_ksZipFileAreaRootDir;
+ #self.oTestSetLogic = TestSetLogic(TMDatabaseConnection(self.dprint if self.fVerbose else None));
+ self.oTestSetLogic = TestSetLogic(TMDatabaseConnection(None));
+ self.fDryRun = oOptions.fDryRun;
+
+ def dprint(self, sText):
+ """ Verbose output. """
+ if self.fVerbose:
+ print(sText);
+ return True;
+
+ def warning(self, sText):
+ """Prints a warning."""
+ print(sText);
+ return True;
+
+ def _processTestSet(self, idTestSet, asFiles, sCurDir):
+ """
+ Worker for processDir.
+ Same return codes as processDir.
+ """
+
+ sBaseFilename = os.path.join(sCurDir, 'TestSet-%d' % (idTestSet,));
+ if sBaseFilename[0:2] == ('.' + os.path.sep):
+ sBaseFilename = sBaseFilename[2:];
+ sSrcFileBase = os.path.join(self.sSrcDir, sBaseFilename + '-');
+
+ #
+ # Skip the file if the test set is still running.
+ # But delete them if the testset is not found.
+ #
+ oTestSet = self.oTestSetLogic.tryFetch(idTestSet);
+ if oTestSet is not None and sBaseFilename != oTestSet.sBaseFilename:
+ self.warning('TestSet %d: Deleting because sBaseFilename differs: "%s" (disk) vs "%s" (db)' \
+ % (idTestSet, sBaseFilename, oTestSet.sBaseFilename,));
+ oTestSet = None;
+
+ if oTestSet is not None:
+ if oTestSet.enmStatus == TestSetData.ksTestStatus_Running:
+ self.dprint('Skipping test set #%d, still running' % (idTestSet,));
+ return True;
+
+ #
+ # If we have a zip file already, don't try recreate it as we might
+ # have had trouble removing the source files.
+ #
+ sDstDirPath = os.path.join(self.sDstDir, sCurDir);
+ sZipFileNm = os.path.join(sDstDirPath, 'TestSet-%d.zip' % (idTestSet,));
+ if not os.path.exists(sZipFileNm):
+ #
+ # Create zip file with all testset files as members.
+ #
+ self.dprint('TestSet %d: Creating %s...' % (idTestSet, sZipFileNm,));
+ if not self.fDryRun:
+
+ if not os.path.exists(sDstDirPath):
+ os.makedirs(sDstDirPath, 0o755);
+
+ utils.noxcptDeleteFile(sZipFileNm + '.tmp');
+ oZipFile = zipfile.ZipFile(sZipFileNm + '.tmp', 'w', zipfile.ZIP_DEFLATED, allowZip64 = True);
+
+ for sFile in asFiles:
+ sSuff = os.path.splitext(sFile)[1];
+ if sSuff in [ '.png', '.webm', '.gz', '.bz2', '.zip', '.mov', '.avi', '.mpg', '.gif', '.jpg' ]:
+ ## @todo Consider storing these files outside the zip if they are a little largish.
+ self.dprint('TestSet %d: Storing %s...' % (idTestSet, sFile));
+ oZipFile.write(sSrcFileBase + sFile, sFile, zipfile.ZIP_STORED);
+ else:
+ self.dprint('TestSet %d: Deflating %s...' % (idTestSet, sFile));
+ oZipFile.write(sSrcFileBase + sFile, sFile, zipfile.ZIP_DEFLATED);
+
+ oZipFile.close();
+
+ #
+ # .zip.tmp -> .zip.
+ #
+ utils.noxcptDeleteFile(sZipFileNm);
+ os.rename(sZipFileNm + '.tmp', sZipFileNm);
+
+ #else: Dry run.
+ else:
+ self.dprint('TestSet %d: zip file exists already (%s)' % (idTestSet, sZipFileNm,));
+
+ #
+ # Delete the files.
+ #
+ fRc = True;
+ if self.fVerbose:
+ self.dprint('TestSet %d: deleting file: %s' % (idTestSet, asFiles));
+ if not self.fDryRun:
+ for sFile in asFiles:
+ if utils.noxcptDeleteFile(sSrcFileBase + sFile) is False:
+ self.warning('TestSet %d: Failed to delete "%s" (%s)' % (idTestSet, sFile, sSrcFileBase + sFile,));
+ fRc = False;
+
+ return fRc;
+
+
+ def processDir(self, sCurDir):
+ """
+ Process the given directory (relative to sSrcDir and sDstDir).
+ Returns success indicator.
+ """
+ if self.fVerbose:
+ self.dprint('processDir: %s' % (sCurDir,));
+
+ #
+ # Sift thought the directory content, collecting subdirectories and
+ # sort relevant files by test set.
+ # Generally there will either be subdirs or there will be files.
+ #
+ asSubDirs = [];
+ dTestSets = {};
+ sCurPath = os.path.abspath(os.path.join(self.sSrcDir, sCurDir));
+ for sFile in os.listdir(sCurPath):
+ if os.path.isdir(os.path.join(sCurPath, sFile)):
+ if sFile not in [ '.', '..' ]:
+ asSubDirs.append(sFile);
+ elif sFile.startswith('TestSet-'):
+ # Parse the file name. ASSUMES 'TestSet-%d-filename' format.
+ iSlash1 = sFile.find('-');
+ iSlash2 = sFile.find('-', iSlash1 + 1);
+ if iSlash2 <= iSlash1:
+ self.warning('Bad filename (1): "%s"' % (sFile,));
+ continue;
+
+ try: idTestSet = int(sFile[(iSlash1 + 1):iSlash2]);
+ except:
+ self.warning('Bad filename (2): "%s"' % (sFile,));
+ if self.fVerbose:
+ self.dprint('\n'.join(utils.getXcptInfo(4)));
+ continue;
+
+ if idTestSet <= 0:
+ self.warning('Bad filename (3): "%s"' % (sFile,));
+ continue;
+
+ if iSlash2 + 2 >= len(sFile):
+ self.warning('Bad filename (4): "%s"' % (sFile,));
+ continue;
+ sName = sFile[(iSlash2 + 1):];
+
+ # Add it.
+ if idTestSet not in dTestSets:
+ dTestSets[idTestSet] = [];
+ asTestSet = dTestSets[idTestSet];
+ asTestSet.append(sName);
+
+ #
+ # Test sets.
+ #
+ fRc = True;
+ for idTestSet in dTestSets:
+ try:
+ if self._processTestSet(idTestSet, dTestSets[idTestSet], sCurDir) is not True:
+ fRc = False;
+ except:
+ self.warning('TestSet %d: Exception in _processTestSet:\n%s' % (idTestSet, '\n'.join(utils.getXcptInfo()),));
+ fRc = False;
+
+ #
+ # Sub dirs.
+ #
+ for sSubDir in asSubDirs:
+ if self.processDir(os.path.join(sCurDir, sSubDir)) is not True:
+ fRc = False;
+
+ #
+ # Try Remove the directory iff it's not '.' and it's been unmodified
+ # for the last 24h (race protection).
+ #
+ if sCurDir != '.':
+ try:
+ fpModTime = float(os.path.getmtime(sCurPath));
+ if fpModTime + (24*3600) <= time.time():
+ if utils.noxcptRmDir(sCurPath) is True:
+ self.dprint('Removed "%s".' % (sCurPath,));
+ except:
+ pass;
+
+ return fRc;
+
+ @staticmethod
+ def main():
+ """ C-style main(). """
+ #
+ # Parse options.
+ #
+ oParser = OptionParser();
+ oParser.add_option('-v', '--verbose', dest = 'fVerbose', action = 'store_true', default = False,
+ help = 'Verbose output.');
+ oParser.add_option('-q', '--quiet', dest = 'fVerbose', action = 'store_false', default = False,
+ help = 'Quiet operation.');
+ oParser.add_option('-d', '--dry-run', dest = 'fDryRun', action = 'store_true', default = False,
+ help = 'Dry run, do not make any changes.');
+ (oOptions, asArgs) = oParser.parse_args()
+ if asArgs != []:
+ oParser.print_help();
+ return 1;
+
+ #
+ # Do the work.
+ #
+ oBatchJob = FileArchiverBatchJob(oOptions);
+ fRc = oBatchJob.processDir('.');
+ return 0 if fRc is True else 1;
+
+if __name__ == '__main__':
+ sys.exit(FileArchiverBatchJob.main());
+
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-13 17:45:32 +0000