diff options
| author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-11 08:17:27 +0000 |
|---|---|---|
| committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-11 08:17:27 +0000 |
| commit | f215e02bf85f68d3a6106c2a1f4f7f063f819064 (patch) | |
| tree | 6bb5b92c046312c4e95ac2620b10ddf482d3fa8b /src/VBox/ValidationKit/testmanager/core | |
| parent | Initial commit. (diff) | |
| download | virtualbox-f215e02bf85f68d3a6106c2a1f4f7f063f819064.tar.xz virtualbox-f215e02bf85f68d3a6106c2a1f4f7f063f819064.zip | |
Adding upstream version 7.0.14-dfsg.upstream/7.0.14-dfsg
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/VBox/ValidationKit/testmanager/core')
38 files changed, 23402 insertions, 0 deletions
diff --git a/src/VBox/ValidationKit/testmanager/core/Makefile.kmk b/src/VBox/ValidationKit/testmanager/core/Makefile.kmk new file mode 100644 index 00000000..74d882cc --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/Makefile.kmk @@ -0,0 +1,46 @@ +# $Id: Makefile.kmk $ +## @file +# VirtualBox Validation Kit. +# + +# +# Copyright (C) 2006-2023 Oracle and/or its affiliates. +# +# This file is part of VirtualBox base platform packages, as +# available from https://www.virtualbox.org. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation, in version 3 of the +# License. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see <https://www.gnu.org/licenses>. +# +# The contents of this file may alternatively be used under the terms +# of the Common Development and Distribution License Version 1.0 +# (CDDL), a copy of it is provided in the "COPYING.CDDL" file included +# in the VirtualBox distribution, in which case the provisions of the +# CDDL are applicable instead of those of the GPL. +# +# You may elect to license modified versions of this file under the +# terms and conditions of either the GPL or the CDDL or both. +# +# SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +# + +SUB_DEPTH = ../../../../.. +include $(KBUILD_PATH)/subheader.kmk + + +VBOX_VALIDATIONKIT_PYTHON_SOURCES += $(wildcard $(PATH_SUB_CURRENT)/*.py) + +$(evalcall def_vbox_validationkit_process_python_sources) +$(evalcall def_vbox_validationkit_process_js_sources) +include $(FILE_KBUILD_SUB_FOOTER) + diff --git a/src/VBox/ValidationKit/testmanager/core/__init__.py b/src/VBox/ValidationKit/testmanager/core/__init__.py new file mode 100644 index 00000000..bfcc7f8b --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/__init__.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# $Id: __init__.py $ + +""" +TestBox Script - Core Logic. +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + diff --git a/src/VBox/ValidationKit/testmanager/core/base.py b/src/VBox/ValidationKit/testmanager/core/base.py new file mode 100755 index 00000000..eac3d921 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/base.py @@ -0,0 +1,1514 @@ +# -*- coding: utf-8 -*- +# $Id: base.py $ +# pylint: disable=too-many-lines + +""" +Test Manager Core - Base Class(es). +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Standard python imports. +import copy; +import datetime; +import json; +import re; +import socket; +import sys; +import uuid; +import unittest; + +# Validation Kit imports. +from common import utils; + +# Python 3 hacks: +if sys.version_info[0] >= 3: + long = int # pylint: disable=redefined-builtin,invalid-name + + +class TMExceptionBase(Exception): + """ + For exceptions raised by any TestManager component. + """ + pass; # pylint: disable=unnecessary-pass + + +class TMTooManyRows(TMExceptionBase): + """ + Too many rows in the result. + Used by ModelLogicBase decendants. + """ + pass; # pylint: disable=unnecessary-pass + + +class TMRowNotFound(TMExceptionBase): + """ + Database row not found. + Used by ModelLogicBase decendants. + """ + pass; # pylint: disable=unnecessary-pass + + +class TMRowAlreadyExists(TMExceptionBase): + """ + Database row already exists (typically raised by addEntry). + Used by ModelLogicBase decendants. + """ + pass; # pylint: disable=unnecessary-pass + + +class TMInvalidData(TMExceptionBase): + """ + Data validation failed. + Used by ModelLogicBase decendants. + """ + pass; # pylint: disable=unnecessary-pass + + +class TMRowInUse(TMExceptionBase): + """ + Database row is in use and cannot be deleted. + Used by ModelLogicBase decendants. + """ + pass; # pylint: disable=unnecessary-pass + + +class TMInFligthCollision(TMExceptionBase): + """ + Database update failed because someone else had already made changes to + the data there. + Used by ModelLogicBase decendants. + """ + pass; # pylint: disable=unnecessary-pass + + +class ModelBase(object): # pylint: disable=too-few-public-methods + """ + Something all classes in the logical model inherits from. + + Not sure if 'logical model' is the right term here. + Will see if it has any purpose later on... + """ + + def __init__(self): + pass; + + +class ModelDataBase(ModelBase): # pylint: disable=too-few-public-methods + """ + Something all classes in the data classes in the logical model inherits from. + """ + + ## Child classes can use this to list array attributes which should use + # an empty array ([]) instead of None as database NULL value. + kasAltArrayNull = []; + + ## validate + ## @{ + ksValidateFor_Add = 'add'; + ksValidateFor_AddForeignId = 'add-foreign-id'; + ksValidateFor_Edit = 'edit'; + ksValidateFor_Other = 'other'; + ## @} + + + ## List of internal attributes which should be ignored by + ## getDataAttributes and related machinery + kasInternalAttributes = []; + + def __init__(self): + ModelBase.__init__(self); + + + # + # Standard methods implemented by combining python magic and hungarian prefixes. + # + + def getDataAttributes(self): + """ + Returns a list of data attributes. + """ + asRet = []; + asAttrs = dir(self); + for sAttr in asAttrs: + if sAttr[0] == '_' or sAttr[0] == 'k': + continue; + if sAttr in self.kasInternalAttributes: + continue; + oValue = getattr(self, sAttr); + if callable(oValue): + continue; + asRet.append(sAttr); + return asRet; + + def initFromOther(self, oOther): + """ + Initialize this object with the values from another instance (child + class instance is accepted). + + This serves as a kind of copy constructor. + + Returns self. May raise exception if the type of other object differs + or is damaged. + """ + for sAttr in self.getDataAttributes(): + setattr(self, sAttr, getattr(oOther, sAttr)); + return self; + + @staticmethod + def getHungarianPrefix(sName): + """ + Returns the hungarian prefix of the given name. + """ + for i, _ in enumerate(sName): + if sName[i] not in ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', + 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z']: + assert re.search('^[A-Z][a-zA-Z0-9]*$', sName[i:]) is not None; + return sName[:i]; + return sName; + + def getAttributeParamNullValues(self, sAttr): + """ + Returns a list of parameter NULL values, with the preferred one being + the first element. + + Child classes can override this to handle one or more attributes specially. + """ + sPrefix = self.getHungarianPrefix(sAttr); + if sPrefix in ['id', 'uid', 'i', 'off', 'pct']: + return [-1, '', '-1',]; + if sPrefix in ['l', 'c',]: + return [long(-1), '', '-1',]; + if sPrefix == 'f': + return ['',]; + if sPrefix in ['enm', 'ip', 's', 'ts', 'uuid']: + return ['',]; + if sPrefix in ['ai', 'aid', 'al', 'as']: + return [[], '', None]; ## @todo ?? + if sPrefix == 'bm': + return ['', [],]; ## @todo bitmaps. + raise TMExceptionBase('Unable to classify "%s" (prefix %s)' % (sAttr, sPrefix)); + + def isAttributeNull(self, sAttr, oValue): + """ + Checks if the specified attribute value indicates NULL. + Return True/False. + + Note! This isn't entirely kosher actually. + """ + if oValue is None: + return True; + aoNilValues = self.getAttributeParamNullValues(sAttr); + return oValue in aoNilValues; + + def _convertAttributeFromParamNull(self, sAttr, oValue): + """ + Converts an attribute from parameter NULL to database NULL value. + Returns the new attribute value. + """ + aoNullValues = self.getAttributeParamNullValues(sAttr); + if oValue in aoNullValues: + oValue = None if sAttr not in self.kasAltArrayNull else []; + # + # Perform deep conversion on ModelDataBase object and lists of them. + # + elif isinstance(oValue, list) and oValue and isinstance(oValue[0], ModelDataBase): + oValue = copy.copy(oValue); + for i, _ in enumerate(oValue): + assert isinstance(oValue[i], ModelDataBase); + oValue[i] = copy.copy(oValue[i]); + oValue[i].convertFromParamNull(); + + elif isinstance(oValue, ModelDataBase): + oValue = copy.copy(oValue); + oValue.convertFromParamNull(); + + return oValue; + + def convertFromParamNull(self): + """ + Converts from parameter NULL values to database NULL values (None). + Returns self. + """ + for sAttr in self.getDataAttributes(): + oValue = getattr(self, sAttr); + oNewValue = self._convertAttributeFromParamNull(sAttr, oValue); + if oValue != oNewValue: + setattr(self, sAttr, oNewValue); + return self; + + def _convertAttributeToParamNull(self, sAttr, oValue): + """ + Converts an attribute from database NULL to a sepcial value we can pass + thru parameter list. + Returns the new attribute value. + """ + if oValue is None: + oValue = self.getAttributeParamNullValues(sAttr)[0]; + # + # Perform deep conversion on ModelDataBase object and lists of them. + # + elif isinstance(oValue, list) and oValue and isinstance(oValue[0], ModelDataBase): + oValue = copy.copy(oValue); + for i, _ in enumerate(oValue): + assert isinstance(oValue[i], ModelDataBase); + oValue[i] = copy.copy(oValue[i]); + oValue[i].convertToParamNull(); + + elif isinstance(oValue, ModelDataBase): + oValue = copy.copy(oValue); + oValue.convertToParamNull(); + + return oValue; + + def convertToParamNull(self): + """ + Converts from database NULL values (None) to special values we can + pass thru parameters list. + Returns self. + """ + for sAttr in self.getDataAttributes(): + oValue = getattr(self, sAttr); + oNewValue = self._convertAttributeToParamNull(sAttr, oValue); + if oValue != oNewValue: + setattr(self, sAttr, oNewValue); + return self; + + def _validateAndConvertAttribute(self, sAttr, sParam, oValue, aoNilValues, fAllowNull, oDb): + """ + Validates and convert one attribute. + Returns the converted value. + + Child classes can override this to handle one or more attributes specially. + Note! oDb can be None. + """ + sPrefix = self.getHungarianPrefix(sAttr); + + if sPrefix in ['id', 'uid']: + (oNewValue, sError) = self.validateInt( oValue, aoNilValues = aoNilValues, fAllowNull = fAllowNull); + elif sPrefix in ['i', 'off', 'pct']: + (oNewValue, sError) = self.validateInt( oValue, aoNilValues = aoNilValues, fAllowNull = fAllowNull, + iMin = getattr(self, 'kiMin_' + sAttr, 0), + iMax = getattr(self, 'kiMax_' + sAttr, 0x7ffffffe)); + elif sPrefix in ['l', 'c']: + (oNewValue, sError) = self.validateLong(oValue, aoNilValues = aoNilValues, fAllowNull = fAllowNull, + lMin = getattr(self, 'klMin_' + sAttr, 0), + lMax = getattr(self, 'klMax_' + sAttr, None)); + elif sPrefix == 'f': + if not oValue and not fAllowNull: oValue = '0'; # HACK ALERT! Checkboxes are only added when checked. + (oNewValue, sError) = self.validateBool(oValue, aoNilValues = aoNilValues, fAllowNull = fAllowNull); + elif sPrefix == 'ts': + (oNewValue, sError) = self.validateTs( oValue, aoNilValues = aoNilValues, fAllowNull = fAllowNull); + elif sPrefix == 'ip': + (oNewValue, sError) = self.validateIp( oValue, aoNilValues = aoNilValues, fAllowNull = fAllowNull); + elif sPrefix == 'uuid': + (oNewValue, sError) = self.validateUuid(oValue, aoNilValues = aoNilValues, fAllowNull = fAllowNull); + elif sPrefix == 'enm': + (oNewValue, sError) = self.validateWord(oValue, aoNilValues = aoNilValues, fAllowNull = fAllowNull, + asValid = getattr(self, 'kasValidValues_' + sAttr)); # The list is required. + elif sPrefix == 's': + (oNewValue, sError) = self.validateStr( oValue, aoNilValues = aoNilValues, fAllowNull = fAllowNull, + cchMin = getattr(self, 'kcchMin_' + sAttr, 0), + cchMax = getattr(self, 'kcchMax_' + sAttr, 4096), + fAllowUnicodeSymbols = getattr(self, 'kfAllowUnicode_' + sAttr, False) ); + ## @todo al. + elif sPrefix == 'aid': + (oNewValue, sError) = self.validateListOfInts(oValue, aoNilValues = aoNilValues, fAllowNull = fAllowNull, + iMin = 1, iMax = 0x7ffffffe); + elif sPrefix == 'as': + (oNewValue, sError) = self.validateListOfStr(oValue, aoNilValues = aoNilValues, fAllowNull = fAllowNull, + asValidValues = getattr(self, 'kasValidValues_' + sAttr, None), + cchMin = getattr(self, 'kcchMin_' + sAttr, 0 if fAllowNull else 1), + cchMax = getattr(self, 'kcchMax_' + sAttr, 4096)); + + elif sPrefix == 'bm': + ## @todo figure out bitfields. + (oNewValue, sError) = self.validateListOfStr(oValue, aoNilValues = aoNilValues, fAllowNull = fAllowNull); + else: + raise TMExceptionBase('Unable to classify "%s" (prefix %s)' % (sAttr, sPrefix)); + + _ = sParam; _ = oDb; + return (oNewValue, sError); + + def _validateAndConvertWorker(self, asAllowNullAttributes, oDb, enmValidateFor = ksValidateFor_Other): + """ + Worker for implementing validateAndConvert(). + """ + dErrors = {}; + for sAttr in self.getDataAttributes(): + oValue = getattr(self, sAttr); + sParam = getattr(self, 'ksParam_' + sAttr); + aoNilValues = self.getAttributeParamNullValues(sAttr); + aoNilValues.append(None); + + (oNewValue, sError) = self._validateAndConvertAttribute(sAttr, sParam, oValue, aoNilValues, + sAttr in asAllowNullAttributes, oDb); + if oValue != oNewValue: + setattr(self, sAttr, oNewValue); + if sError is not None: + dErrors[sParam] = sError; + + # Check the NULL requirements of the primary ID(s) for the 'add' and 'edit' actions. + if enmValidateFor in (ModelDataBase.ksValidateFor_Add, + ModelDataBase.ksValidateFor_AddForeignId, + ModelDataBase.ksValidateFor_Edit,): + fMustBeNull = enmValidateFor == ModelDataBase.ksValidateFor_Add; + sAttr = getattr(self, 'ksIdAttr', None); + if sAttr is not None: + oValue = getattr(self, sAttr); + if self.isAttributeNull(sAttr, oValue) != fMustBeNull: + sParam = getattr(self, 'ksParam_' + sAttr); + sErrMsg = 'Must be NULL!' if fMustBeNull else 'Must not be NULL!' + if sParam in dErrors: + dErrors[sParam] += ' ' + sErrMsg; + else: + dErrors[sParam] = sErrMsg; + + return dErrors; + + def validateAndConvert(self, oDb, enmValidateFor = ksValidateFor_Other): + """ + Validates the input and converts valid fields to their right type. + Returns a dictionary with per field reports, only invalid fields will + be returned, so an empty dictionary means that the data is valid. + + The dictionary keys are ksParam_*. + + Child classes can override _validateAndConvertAttribute to handle + selected fields specially. There are also a few class variables that + can be used to advice the validation: kcchMin_sAttr, kcchMax_sAttr, + kiMin_iAttr, kiMax_iAttr, klMin_lAttr, klMax_lAttr, + kasValidValues_enmAttr, and kasAllowNullAttributes. + """ + return self._validateAndConvertWorker(getattr(self, 'kasAllowNullAttributes', []), oDb, + enmValidateFor = enmValidateFor); + + def validateAndConvertEx(self, asAllowNullAttributes, oDb, enmValidateFor = ksValidateFor_Other): + """ + Same as validateAndConvert but with custom allow-null list. + """ + return self._validateAndConvertWorker(asAllowNullAttributes, oDb, enmValidateFor = enmValidateFor); + + def convertParamToAttribute(self, sAttr, sParam, oValue, oDisp, fStrict): + """ + Calculate the attribute value when initialized from a parameter. + + Returns the new value, with parameter NULL values. Raises exception on + invalid parameter value. + + Child classes can override to do special parameter conversion jobs. + """ + sPrefix = self.getHungarianPrefix(sAttr); + asValidValues = getattr(self, 'kasValidValues_' + sAttr, None); + fAllowNull = sAttr in getattr(self, 'kasAllowNullAttributes', []); + if fStrict: + if sPrefix == 'f': + # HACK ALERT! Checkboxes are only present when checked, so we always have to provide a default. + oNewValue = oDisp.getStringParam(sParam, asValidValues, '0'); + elif sPrefix[0] == 'a': + # HACK ALERT! Lists are not present if empty. + oNewValue = oDisp.getListOfStrParams(sParam, []); + else: + oNewValue = oDisp.getStringParam(sParam, asValidValues, None, fAllowNull = fAllowNull); + else: + if sPrefix[0] == 'a': + oNewValue = oDisp.getListOfStrParams(sParam, []); + else: + assert oValue is not None, 'sAttr=%s' % (sAttr,); + oNewValue = oDisp.getStringParam(sParam, asValidValues, oValue, fAllowNull = fAllowNull); + return oNewValue; + + def initFromParams(self, oDisp, fStrict = True): + """ + Initialize the object from parameters. + The input is not validated at all, except that all parameters must be + present when fStrict is True. + + Returns self. Raises exception on invalid parameter value. + + Note! The returned object has parameter NULL values, not database ones! + """ + + self.convertToParamNull() + for sAttr in self.getDataAttributes(): + oValue = getattr(self, sAttr); + oNewValue = self.convertParamToAttribute(sAttr, getattr(self, 'ksParam_' + sAttr), oValue, oDisp, fStrict); + if oNewValue != oValue: + setattr(self, sAttr, oNewValue); + return self; + + def areAttributeValuesEqual(self, sAttr, sPrefix, oValue1, oValue2): + """ + Called to compare two attribute values and python thinks differs. + + Returns True/False. + + Child classes can override this to do special compares of things like arrays. + """ + # Just in case someone uses it directly. + if oValue1 == oValue2: + return True; + + # + # Timestamps can be both string (param) and object (db) + # depending on the data source. Compare string values to make + # sure we're doing the right thing here. + # + if sPrefix == 'ts': + return str(oValue1) == str(oValue2); + + # + # Some generic code handling ModelDataBase children. + # + if isinstance(oValue1, list) and isinstance(oValue2, list): + if len(oValue1) == len(oValue2): + for i, _ in enumerate(oValue1): + if not isinstance(oValue1[i], ModelDataBase) \ + or type(oValue1) is not type(oValue2): + return False; + if not oValue1[i].isEqual(oValue2[i]): + return False; + return True; + + elif isinstance(oValue1, ModelDataBase) \ + and type(oValue1) is type(oValue2): + return oValue1[i].isEqual(oValue2[i]); + + _ = sAttr; + return False; + + def isEqual(self, oOther): + """ Compares two instances. """ + for sAttr in self.getDataAttributes(): + if getattr(self, sAttr) != getattr(oOther, sAttr): + # Delegate the final decision to an overridable method. + if not self.areAttributeValuesEqual(sAttr, self.getHungarianPrefix(sAttr), + getattr(self, sAttr), getattr(oOther, sAttr)): + return False; + return True; + + def isEqualEx(self, oOther, asExcludeAttrs): + """ Compares two instances, omitting the given attributes. """ + for sAttr in self.getDataAttributes(): + if sAttr not in asExcludeAttrs \ + and getattr(self, sAttr) != getattr(oOther, sAttr): + # Delegate the final decision to an overridable method. + if not self.areAttributeValuesEqual(sAttr, self.getHungarianPrefix(sAttr), + getattr(self, sAttr), getattr(oOther, sAttr)): + return False; + return True; + + def reinitToNull(self): + """ + Reinitializes the object to (database) NULL values. + Returns self. + """ + for sAttr in self.getDataAttributes(): + setattr(self, sAttr, None); + return self; + + def toString(self): + """ + Stringifies the object. + Returns string representation. + """ + + sMembers = ''; + for sAttr in self.getDataAttributes(): + oValue = getattr(self, sAttr); + sMembers += ', %s=%s' % (sAttr, oValue); + + oClass = type(self); + if sMembers == '': + return '<%s>' % (oClass.__name__); + return '<%s: %s>' % (oClass.__name__, sMembers[2:]); + + def __str__(self): + return self.toString(); + + + + # + # New validation helpers. + # + # These all return (oValue, sError), where sError is None when the value + # is valid and an error message when not. On success and in case of + # range errors, oValue is converted into the requested type. + # + + @staticmethod + def validateInt(sValue, iMin = 0, iMax = 0x7ffffffe, aoNilValues = tuple([-1, None, '']), fAllowNull = True): + """ Validates an integer field. """ + if sValue in aoNilValues: + if fAllowNull: + return (None if sValue is None else aoNilValues[0], None); + return (sValue, 'Mandatory.'); + + try: + if utils.isString(sValue): + iValue = int(sValue, 0); + else: + iValue = int(sValue); + except: + return (sValue, 'Not an integer'); + + if iValue in aoNilValues: + return (aoNilValues[0], None if fAllowNull else 'Mandatory.'); + + if iValue < iMin: + return (iValue, 'Value too small (min %d)' % (iMin,)); + if iValue > iMax: + return (iValue, 'Value too high (max %d)' % (iMax,)); + return (iValue, None); + + @staticmethod + def validateLong(sValue, lMin = 0, lMax = None, aoNilValues = tuple([long(-1), None, '']), fAllowNull = True): + """ Validates an long integer field. """ + if sValue in aoNilValues: + if fAllowNull: + return (None if sValue is None else aoNilValues[0], None); + return (sValue, 'Mandatory.'); + try: + if utils.isString(sValue): + lValue = long(sValue, 0); + else: + lValue = long(sValue); + except: + return (sValue, 'Not a long integer'); + + if lValue in aoNilValues: + return (aoNilValues[0], None if fAllowNull else 'Mandatory.'); + + if lMin is not None and lValue < lMin: + return (lValue, 'Value too small (min %d)' % (lMin,)); + if lMax is not None and lValue > lMax: + return (lValue, 'Value too high (max %d)' % (lMax,)); + return (lValue, None); + + kdTimestampRegex = { + len('2012-10-08 01:54:06'): r'(\d{4})-([01]\d)-([0123]\d)[ Tt]([012]\d):[0-5]\d:([0-6]\d)$', + len('2012-10-08 01:54:06.00'): r'(\d{4})-([01]\d)-([0123]\d)[ Tt]([012]\d):[0-5]\d:([0-6]\d).\d{2}$', + len('2012-10-08 01:54:06.000'): r'(\d{4})-([01]\d)-([0123]\d)[ Tt]([012]\d):[0-5]\d:([0-6]\d).\d{3}$', + len('999999-12-31 00:00:00.00'): r'(\d{6})-([01]\d)-([0123]\d)[ Tt]([012]\d):[0-5]\d:([0-6]\d).\d{2}$', + len('9999-12-31 23:59:59.999999'): r'(\d{4})-([01]\d)-([0123]\d)[ Tt]([012]\d):[0-5]\d:([0-6]\d).\d{6}$', + len('9999-12-31T23:59:59.999999999'): r'(\d{4})-([01]\d)-([0123]\d)[ Tt]([012]\d):[0-5]\d:([0-6]\d).\d{9}$', + }; + + @staticmethod + def validateTs(sValue, aoNilValues = tuple([None, '']), fAllowNull = True, fRelative = False): + """ Validates a timestamp field. """ + if sValue in aoNilValues: + return (sValue, None if fAllowNull else 'Mandatory.'); + if not utils.isString(sValue): + return (sValue, None); + + # Validate and strip off the timezone stuff. + if sValue[-1] in 'Zz': + sStripped = sValue[:-1]; + sValue = sStripped + 'Z'; + elif len(sValue) >= 19 + 3: + oRes = re.match(r'^.*[+-](\d\d):(\d\d)$', sValue); + if oRes is not None: + if int(oRes.group(1)) > 12 or int(oRes.group(2)) >= 60: + return (sValue, 'Invalid timezone offset.'); + sStripped = sValue[:-6]; + else: + sStripped = sValue; + else: + sStripped = sValue; + + # Used the stripped value length to find regular expression for validating and parsing the timestamp. + sError = None; + sRegExp = ModelDataBase.kdTimestampRegex.get(len(sStripped), None); + if sRegExp: + oRes = re.match(sRegExp, sStripped); + if oRes is not None: + iYear = int(oRes.group(1)); + if iYear % 4 == 0 and (iYear % 100 != 0 or iYear % 400 == 0): + acDaysOfMonth = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; + else: + acDaysOfMonth = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; + iMonth = int(oRes.group(2)); + iDay = int(oRes.group(3)); + iHour = int(oRes.group(4)); + iSec = int(oRes.group(5)); + if iMonth > 12 or (iMonth <= 0 and not fRelative): + sError = 'Invalid timestamp month.'; + elif iDay > acDaysOfMonth[iMonth - 1]: + sError = 'Invalid timestamp day-of-month (%02d has %d days).' % (iMonth, acDaysOfMonth[iMonth - 1]); + elif iHour > 23: + sError = 'Invalid timestamp hour.' + elif iSec >= 61: + sError = 'Invalid timestamp second.' + elif iSec >= 60: + sError = 'Invalid timestamp: no leap seconds, please.' + else: + sError = 'Invalid timestamp (validation regexp: %s).' % (sRegExp,); + else: + sError = 'Invalid timestamp length.'; + return (sValue, sError); + + @staticmethod + def validateIp(sValue, aoNilValues = tuple([None, '']), fAllowNull = True): + """ Validates an IP address field. """ + if sValue in aoNilValues: + return (sValue, None if fAllowNull else 'Mandatory.'); + + if sValue == '::1': + return (sValue, None); + + try: + socket.inet_pton(socket.AF_INET, sValue); # pylint: disable=no-member + except: + try: + socket.inet_pton(socket.AF_INET6, sValue); # pylint: disable=no-member + except: + return (sValue, 'Not a valid IP address.'); + + return (sValue, None); + + @staticmethod + def validateBool(sValue, aoNilValues = tuple([None, '']), fAllowNull = True): + """ Validates a boolean field. """ + if sValue in aoNilValues: + return (sValue, None if fAllowNull else 'Mandatory.'); + + if sValue in ('True', 'true', '1', True): + return (True, None); + if sValue in ('False', 'false', '0', False): + return (False, None); + return (sValue, 'Invalid boolean value.'); + + @staticmethod + def validateUuid(sValue, aoNilValues = tuple([None, '']), fAllowNull = True): + """ Validates an UUID field. """ + if sValue in aoNilValues: + return (sValue, None if fAllowNull else 'Mandatory.'); + + try: + sValue = str(uuid.UUID(sValue)); + except: + return (sValue, 'Invalid UUID value.'); + return (sValue, None); + + @staticmethod + def validateWord(sValue, cchMin = 1, cchMax = 64, asValid = None, aoNilValues = tuple([None, '']), fAllowNull = True): + """ Validates a word field. """ + if sValue in aoNilValues: + return (sValue, None if fAllowNull else 'Mandatory.'); + + if re.search('[^a-zA-Z0-9_-]', sValue) is not None: + sError = 'Single word ([a-zA-Z0-9_-]), please.'; + elif cchMin is not None and len(sValue) < cchMin: + sError = 'Too short, min %s chars' % (cchMin,); + elif cchMax is not None and len(sValue) > cchMax: + sError = 'Too long, max %s chars' % (cchMax,); + elif asValid is not None and sValue not in asValid: + sError = 'Invalid value "%s", must be one of: %s' % (sValue, asValid); + else: + sError = None; + return (sValue, sError); + + @staticmethod + def validateStr(sValue, cchMin = 0, cchMax = 4096, aoNilValues = tuple([None, '']), fAllowNull = True, + fAllowUnicodeSymbols = False): + """ Validates a string field. """ + if sValue in aoNilValues: + return (sValue, None if fAllowNull else 'Mandatory.'); + + if cchMin is not None and len(sValue) < cchMin: + sError = 'Too short, min %s chars' % (cchMin,); + elif cchMax is not None and len(sValue) > cchMax: + sError = 'Too long, max %s chars' % (cchMax,); + elif fAllowUnicodeSymbols is False and utils.hasNonAsciiCharacters(sValue): + sError = 'Non-ascii characters not allowed' + else: + sError = None; + return (sValue, sError); + + @staticmethod + def validateEmail(sValue, aoNilValues = tuple([None, '']), fAllowNull = True): + """ Validates a email field.""" + if sValue in aoNilValues: + return (sValue, None if fAllowNull else 'Mandatory.'); + + if re.match(r'.+@.+\..+', sValue) is None: + return (sValue,'Invalid e-mail format.'); + return (sValue, None); + + @staticmethod + def validateListOfSomething(asValues, aoNilValues = tuple([[], None]), fAllowNull = True): + """ Validate a list of some uniform values. Returns a copy of the list (if list it is). """ + if asValues in aoNilValues or (not asValues and not fAllowNull): + return (asValues, None if fAllowNull else 'Mandatory.') + + if not isinstance(asValues, list): + return (asValues, 'Invalid data type (%s).' % (type(asValues),)); + + asValues = list(asValues); # copy the list. + if asValues: + oType = type(asValues[0]); + for i in range(1, len(asValues)): + if type(asValues[i]) is not oType: # pylint: disable=unidiomatic-typecheck + return (asValues, 'Invalid entry data type ([0]=%s vs [%d]=%s).' % (oType, i, type(asValues[i])) ); + + return (asValues, None); + + @staticmethod + def validateListOfStr(asValues, cchMin = None, cchMax = None, asValidValues = None, + aoNilValues = tuple([[], None]), fAllowNull = True): + """ Validates a list of text items.""" + (asValues, sError) = ModelDataBase.validateListOfSomething(asValues, aoNilValues, fAllowNull); + + if sError is None and asValues not in aoNilValues and asValues: + if not utils.isString(asValues[0]): + return (asValues, 'Invalid item data type.'); + + if not fAllowNull and cchMin is None: + cchMin = 1; + + for sValue in asValues: + if asValidValues is not None and sValue not in asValidValues: + sThisErr = 'Invalid value "%s".' % (sValue,); + elif cchMin is not None and len(sValue) < cchMin: + sThisErr = 'Value "%s" is too short, min length is %u chars.' % (sValue, cchMin); + elif cchMax is not None and len(sValue) > cchMax: + sThisErr = 'Value "%s" is too long, max length is %u chars.' % (sValue, cchMax); + else: + continue; + + if sError is None: + sError = sThisErr; + else: + sError += ' ' + sThisErr; + + return (asValues, sError); + + @staticmethod + def validateListOfInts(asValues, iMin = 0, iMax = 0x7ffffffe, aoNilValues = tuple([[], None]), fAllowNull = True): + """ Validates a list of integer items.""" + (asValues, sError) = ModelDataBase.validateListOfSomething(asValues, aoNilValues, fAllowNull); + + if sError is None and asValues not in aoNilValues and asValues: + for i, _ in enumerate(asValues): + sValue = asValues[i]; + + sThisErr = ''; + try: + iValue = int(sValue); + except: + sThisErr = 'Invalid integer value "%s".' % (sValue,); + else: + asValues[i] = iValue; + if iValue < iMin: + sThisErr = 'Value %d is too small (min %d)' % (iValue, iMin,); + elif iValue > iMax: + sThisErr = 'Value %d is too high (max %d)' % (iValue, iMax,); + else: + continue; + + if sError is None: + sError = sThisErr; + else: + sError += ' ' + sThisErr; + + return (asValues, sError); + + + + # + # Old validation helpers. + # + + @staticmethod + def _validateInt(dErrors, sName, sValue, iMin = 0, iMax = 0x7ffffffe, aoNilValues = tuple([-1, None, ''])): + """ Validates an integer field. """ + (sValue, sError) = ModelDataBase.validateInt(sValue, iMin, iMax, aoNilValues, fAllowNull = True); + if sError is not None: + dErrors[sName] = sError; + return sValue; + + @staticmethod + def _validateIntNN(dErrors, sName, sValue, iMin = 0, iMax = 0x7ffffffe, aoNilValues = tuple([-1, None, ''])): + """ Validates an integer field, not null. """ + (sValue, sError) = ModelDataBase.validateInt(sValue, iMin, iMax, aoNilValues, fAllowNull = False); + if sError is not None: + dErrors[sName] = sError; + return sValue; + + @staticmethod + def _validateLong(dErrors, sName, sValue, lMin = 0, lMax = None, aoNilValues = tuple([long(-1), None, ''])): + """ Validates an long integer field. """ + (sValue, sError) = ModelDataBase.validateLong(sValue, lMin, lMax, aoNilValues, fAllowNull = False); + if sError is not None: + dErrors[sName] = sError; + return sValue; + + @staticmethod + def _validateLongNN(dErrors, sName, sValue, lMin = 0, lMax = None, aoNilValues = tuple([long(-1), None, ''])): + """ Validates an long integer field, not null. """ + (sValue, sError) = ModelDataBase.validateLong(sValue, lMin, lMax, aoNilValues, fAllowNull = True); + if sError is not None: + dErrors[sName] = sError; + return sValue; + + @staticmethod + def _validateTs(dErrors, sName, sValue): + """ Validates a timestamp field. """ + (sValue, sError) = ModelDataBase.validateTs(sValue, fAllowNull = True); + if sError is not None: + dErrors[sName] = sError; + return sValue; + + @staticmethod + def _validateTsNN(dErrors, sName, sValue): + """ Validates a timestamp field, not null. """ + (sValue, sError) = ModelDataBase.validateTs(sValue, fAllowNull = False); + if sError is not None: + dErrors[sName] = sError; + return sValue; + + @staticmethod + def _validateIp(dErrors, sName, sValue): + """ Validates an IP address field. """ + (sValue, sError) = ModelDataBase.validateIp(sValue, fAllowNull = True); + if sError is not None: + dErrors[sName] = sError; + return sValue; + + @staticmethod + def _validateIpNN(dErrors, sName, sValue): + """ Validates an IP address field, not null. """ + (sValue, sError) = ModelDataBase.validateIp(sValue, fAllowNull = False); + if sError is not None: + dErrors[sName] = sError; + return sValue; + + @staticmethod + def _validateBool(dErrors, sName, sValue): + """ Validates a boolean field. """ + (sValue, sError) = ModelDataBase.validateBool(sValue, fAllowNull = True); + if sError is not None: + dErrors[sName] = sError; + return sValue; + + @staticmethod + def _validateBoolNN(dErrors, sName, sValue): + """ Validates a boolean field, not null. """ + (sValue, sError) = ModelDataBase.validateBool(sValue, fAllowNull = False); + if sError is not None: + dErrors[sName] = sError; + return sValue; + + @staticmethod + def _validateUuid(dErrors, sName, sValue): + """ Validates an UUID field. """ + (sValue, sError) = ModelDataBase.validateUuid(sValue, fAllowNull = True); + if sError is not None: + dErrors[sName] = sError; + return sValue; + + @staticmethod + def _validateUuidNN(dErrors, sName, sValue): + """ Validates an UUID field, not null. """ + (sValue, sError) = ModelDataBase.validateUuid(sValue, fAllowNull = False); + if sError is not None: + dErrors[sName] = sError; + return sValue; + + @staticmethod + def _validateWord(dErrors, sName, sValue, cchMin = 1, cchMax = 64, asValid = None): + """ Validates a word field. """ + (sValue, sError) = ModelDataBase.validateWord(sValue, cchMin, cchMax, asValid, fAllowNull = True); + if sError is not None: + dErrors[sName] = sError; + return sValue; + + @staticmethod + def _validateWordNN(dErrors, sName, sValue, cchMin = 1, cchMax = 64, asValid = None): + """ Validates a boolean field, not null. """ + (sValue, sError) = ModelDataBase.validateWord(sValue, cchMin, cchMax, asValid, fAllowNull = False); + if sError is not None: + dErrors[sName] = sError; + return sValue; + + @staticmethod + def _validateStr(dErrors, sName, sValue, cchMin = 0, cchMax = 4096): + """ Validates a string field. """ + (sValue, sError) = ModelDataBase.validateStr(sValue, cchMin, cchMax, fAllowNull = True); + if sError is not None: + dErrors[sName] = sError; + return sValue; + + @staticmethod + def _validateStrNN(dErrors, sName, sValue, cchMin = 0, cchMax = 4096): + """ Validates a string field, not null. """ + (sValue, sError) = ModelDataBase.validateStr(sValue, cchMin, cchMax, fAllowNull = False); + if sError is not None: + dErrors[sName] = sError; + return sValue; + + @staticmethod + def _validateEmail(dErrors, sName, sValue): + """ Validates a email field.""" + (sValue, sError) = ModelDataBase.validateEmail(sValue, fAllowNull = True); + if sError is not None: + dErrors[sName] = sError; + return sValue; + + @staticmethod + def _validateEmailNN(dErrors, sName, sValue): + """ Validates a email field.""" + (sValue, sError) = ModelDataBase.validateEmail(sValue, fAllowNull = False); + if sError is not None: + dErrors[sName] = sError; + return sValue; + + @staticmethod + def _validateListOfStr(dErrors, sName, asValues, asValidValues = None): + """ Validates a list of text items.""" + (sValue, sError) = ModelDataBase.validateListOfStr(asValues, asValidValues = asValidValues, fAllowNull = True); + if sError is not None: + dErrors[sName] = sError; + return sValue; + + @staticmethod + def _validateListOfStrNN(dErrors, sName, asValues, asValidValues = None): + """ Validates a list of text items, not null and len >= 1.""" + (sValue, sError) = ModelDataBase.validateListOfStr(asValues, asValidValues = asValidValues, fAllowNull = False); + if sError is not None: + dErrors[sName] = sError; + return sValue; + + # + # Various helpers. + # + + @staticmethod + def formatSimpleNowAndPeriod(oDb, tsNow = None, sPeriodBack = None, + sTablePrefix = '', sExpCol = 'tsExpire', sEffCol = 'tsEffective'): + """ + Formats a set of tsNow and sPeriodBack arguments for a standard testmanager + table. + + If sPeriodBack is given, the query is effective for the period + (tsNow - sPeriodBack) thru (tsNow). + + If tsNow isn't given, it defaults to current time. + + Returns the final portion of a WHERE query (start with AND) and maybe an + ORDER BY and LIMIT bit if sPeriodBack is given. + """ + if tsNow is not None: + if sPeriodBack is not None: + sRet = oDb.formatBindArgs(' AND ' + sTablePrefix + sExpCol + ' > (%s::timestamp - %s::interval)\n' + ' AND tsEffective <= %s\n' + 'ORDER BY ' + sTablePrefix + sExpCol + ' DESC\n' + 'LIMIT 1\n' + , ( tsNow, sPeriodBack, tsNow)); + else: + sRet = oDb.formatBindArgs(' AND ' + sTablePrefix + sExpCol + ' > %s\n' + ' AND ' + sTablePrefix + sEffCol + ' <= %s\n' + , ( tsNow, tsNow, )); + else: + if sPeriodBack is not None: + sRet = oDb.formatBindArgs(' AND ' + sTablePrefix + sExpCol + ' > (CURRENT_TIMESTAMP - %s::interval)\n' + ' AND ' + sTablePrefix + sEffCol + ' <= CURRENT_TIMESTAMP\n' + 'ORDER BY ' + sTablePrefix + sExpCol + ' DESC\n' + 'LIMIT 1\n' + , ( sPeriodBack, )); + else: + sRet = ' AND ' + sTablePrefix + sExpCol + ' = \'infinity\'::timestamp\n'; + return sRet; + + @staticmethod + def formatSimpleNowAndPeriodQuery(oDb, sQuery, aBindArgs, tsNow = None, sPeriodBack = None, + sTablePrefix = '', sExpCol = 'tsExpire', sEffCol = 'tsEffective'): + """ + Formats a simple query for a standard testmanager table with optional + tsNow and sPeriodBack arguments. + + The sQuery and sBindArgs are passed along to oDb.formatBindArgs to form + the first part of the query. Must end with an open WHERE statement as + we'll be adding the time part starting with 'AND something...'. + + See formatSimpleNowAndPeriod for tsNow and sPeriodBack description. + + Returns the final portion of a WHERE query (start with AND) and maybe an + ORDER BY and LIMIT bit if sPeriodBack is given. + + """ + return oDb.formatBindArgs(sQuery, aBindArgs) \ + + ModelDataBase.formatSimpleNowAndPeriod(oDb, tsNow, sPeriodBack, sTablePrefix, sExpCol, sEffCol); + + + # + # JSON + # + + @staticmethod + def stringToJson(sString): + """ Converts a string to a JSON value string. """ + if not utils.isString(sString): + sString = utils.toUnicode(sString); + if not utils.isString(sString): + sString = str(sString); + return json.dumps(sString); + + @staticmethod + def dictToJson(dDict, dOptions = None): + """ Converts a dictionary to a JSON string. """ + sJson = u'{ '; + for i, oKey in enumerate(dDict): + if i > 0: + sJson += ', '; + sJson += '%s: %s' % (ModelDataBase.stringToJson(oKey), + ModelDataBase.genericToJson(dDict[oKey], dOptions)); + return sJson + ' }'; + + @staticmethod + def listToJson(aoList, dOptions = None): + """ Converts list of something to a JSON string. """ + sJson = u'[ '; + for i, oValue in enumerate(aoList): + if i > 0: + sJson += u', '; + sJson += ModelDataBase.genericToJson(oValue, dOptions); + return sJson + u' ]'; + + @staticmethod + def datetimeToJson(oDateTime): + """ Converts a datetime instance to a JSON string. """ + return '"%s"' % (oDateTime,); + + + @staticmethod + def genericToJson(oValue, dOptions = None): + """ Converts a generic object to a JSON string. """ + if isinstance(oValue, ModelDataBase): + return oValue.toJson(); + if isinstance(oValue, dict): + return ModelDataBase.dictToJson(oValue, dOptions); + if isinstance(oValue, (list, tuple, set, frozenset)): + return ModelDataBase.listToJson(oValue, dOptions); + if isinstance(oValue, datetime.datetime): + return ModelDataBase.datetimeToJson(oValue) + return json.dumps(oValue); + + def attribValueToJson(self, sAttr, oValue, dOptions = None): + """ + Converts the attribute value to JSON. + Returns JSON (string). + """ + _ = sAttr; + return self.genericToJson(oValue, dOptions); + + def toJson(self, dOptions = None): + """ + Converts the object to JSON. + Returns JSON (string). + """ + sJson = u'{ '; + for iAttr, sAttr in enumerate(self.getDataAttributes()): + oValue = getattr(self, sAttr); + if iAttr > 0: + sJson += ', '; + sJson += u'"%s": ' % (sAttr,); + sJson += self.attribValueToJson(sAttr, oValue, dOptions); + return sJson + u' }'; + + + # + # Sub-classes. + # + + class DispWrapper(object): + """Proxy object.""" + def __init__(self, oDisp, sAttrFmt): + self.oDisp = oDisp; + self.sAttrFmt = sAttrFmt; + def getStringParam(self, sName, asValidValues = None, sDefault = None, fAllowNull = False): + """See WuiDispatcherBase.getStringParam.""" + return self.oDisp.getStringParam(self.sAttrFmt % (sName,), asValidValues, sDefault, fAllowNull = fAllowNull); + def getListOfStrParams(self, sName, asDefaults = None): + """See WuiDispatcherBase.getListOfStrParams.""" + return self.oDisp.getListOfStrParams(self.sAttrFmt % (sName,), asDefaults); + def getListOfIntParams(self, sName, iMin = None, iMax = None, aiDefaults = None): + """See WuiDispatcherBase.getListOfIntParams.""" + return self.oDisp.getListOfIntParams(self.sAttrFmt % (sName,), iMin, iMax, aiDefaults); + + + + +# pylint: disable=no-member,missing-docstring,too-few-public-methods +class ModelDataBaseTestCase(unittest.TestCase): + """ + Base testcase for ModelDataBase decendants. + Derive from this and override setUp. + """ + + def setUp(self): + """ + Override this! Don't call super! + The subclasses are expected to set aoSamples to an array of instance + samples. The first entry must be a default object, the subsequent ones + are optional and their contents freely choosen. + """ + self.aoSamples = [ModelDataBase(),]; + + def testEquality(self): + for oSample in self.aoSamples: + self.assertEqual(oSample.isEqual(copy.copy(oSample)), True); + self.assertIsNotNone(oSample.isEqual(self.aoSamples[0])); + + def testNullConversion(self): + if not self.aoSamples[0].getDataAttributes(): + return; + for oSample in self.aoSamples: + oCopy = copy.copy(oSample); + self.assertEqual(oCopy.convertToParamNull(), oCopy); + self.assertEqual(oCopy.isEqual(oSample), False); + self.assertEqual(oCopy.convertFromParamNull(), oCopy); + self.assertEqual(oCopy.isEqual(oSample), True, '\ngot : %s\nexpected: %s' % (oCopy, oSample,)); + + oCopy = copy.copy(oSample); + self.assertEqual(oCopy.convertToParamNull(), oCopy); + oCopy2 = copy.copy(oCopy); + self.assertEqual(oCopy.convertToParamNull(), oCopy); + self.assertEqual(oCopy.isEqual(oCopy2), True); + self.assertEqual(oCopy.convertToParamNull(), oCopy); + self.assertEqual(oCopy.isEqual(oCopy2), True); + + oCopy = copy.copy(oSample); + self.assertEqual(oCopy.convertFromParamNull(), oCopy); + oCopy2 = copy.copy(oCopy); + self.assertEqual(oCopy.convertFromParamNull(), oCopy); + self.assertEqual(oCopy.isEqual(oCopy2), True); + self.assertEqual(oCopy.convertFromParamNull(), oCopy); + self.assertEqual(oCopy.isEqual(oCopy2), True); + + def testReinitToNull(self): + oFirst = copy.copy(self.aoSamples[0]); + self.assertEqual(oFirst.reinitToNull(), oFirst); + for oSample in self.aoSamples: + oCopy = copy.copy(oSample); + self.assertEqual(oCopy.reinitToNull(), oCopy); + self.assertEqual(oCopy.isEqual(oFirst), True); + + def testValidateAndConvert(self): + for oSample in self.aoSamples: + oCopy = copy.copy(oSample); + oCopy.convertToParamNull(); + dError1 = oCopy.validateAndConvert(None); + + oCopy2 = copy.copy(oCopy); + self.assertEqual(oCopy.validateAndConvert(None), dError1); + self.assertEqual(oCopy.isEqual(oCopy2), True); + + def testInitFromParams(self): + class DummyDisp(object): + def getStringParam(self, sName, asValidValues = None, sDefault = None, fAllowNull = False): + _ = sName; _ = asValidValues; _ = fAllowNull; + return sDefault; + def getListOfStrParams(self, sName, asDefaults = None): + _ = sName; + return asDefaults; + def getListOfIntParams(self, sName, iMin = None, iMax = None, aiDefaults = None): + _ = sName; _ = iMin; _ = iMax; + return aiDefaults; + + for oSample in self.aoSamples: + oCopy = copy.copy(oSample); + self.assertEqual(oCopy.initFromParams(DummyDisp(), fStrict = False), oCopy); + + def testToString(self): + for oSample in self.aoSamples: + self.assertIsNotNone(oSample.toString()); + + +class FilterCriterionValueAndDescription(object): + """ + A filter criterion value and its description. + """ + + def __init__(self, oValue, sDesc, cTimes = None, sHover = None, fIrrelevant = False): + self.oValue = oValue; ##< Typically the ID of something in the database. + self.sDesc = sDesc; ##< What to display. + self.cTimes = cTimes; ##< Number of times the value occurs in the result set. None if not given. + self.sHover = sHover; ##< Optional hover/title string. + self.fIrrelevant = fIrrelevant; ##< Irrelevant filter option, only present because it's selected + self.aoSubs = []; ##< References to FilterCriterion.oSub.aoPossible. + + +class FilterCriterion(object): + """ + A filter criterion. + """ + + ## @name The state. + ## @{ + ksState_NotSelected = 'not-selected'; + ksState_Selected = 'selected'; + ## @} + + ## @name The kind of filtering. + ## @{ + ## 'Element of' by default, 'not an element of' when fInverted is False. + ksKind_ElementOfOrNot = 'element-of-or-not'; + ## The criterion is a special one and cannot be inverted. + ksKind_Special = 'special'; + ## @} + + ## @name The value type. + ## @{ + ksType_UInt = 'uint'; ##< unsigned integer value. + ksType_UIntNil = 'uint-nil'; ##< unsigned integer value, with nil. + ksType_String = 'string'; ##< string value. + ksType_Ranges = 'ranges'; ##< List of (unsigned) integer ranges. + ## @} + + def __init__(self, sName, sVarNm = None, sType = ksType_UInt, # pylint: disable=too-many-arguments + sState = ksState_NotSelected, sKind = ksKind_ElementOfOrNot, + sTable = None, sColumn = None, asTables = None, oSub = None): + assert len(sVarNm) == 2; # required by wuimain.py for filtering. + self.sName = sName; + self.sState = sState; + self.sType = sType; + self.sKind = sKind; + self.sVarNm = sVarNm; + self.aoSelected = []; ##< User input from sVarNm. Single value, type according to sType. + self.sInvVarNm = 'i' + sVarNm if sKind == self.ksKind_ElementOfOrNot else None; + self.fInverted = False; ##< User input from sInvVarNm. Inverts the operation (-> not an element of). + self.aoPossible = []; ##< type: list[FilterCriterionValueAndDescription] + assert (sTable is None and asTables is None) or ((sTable is not None) != (asTables is not None)), \ + '%s %s' % (sTable, asTables); + self.asTables = [sTable,] if sTable is not None else asTables; + assert sColumn is None or len(self.asTables) == 1, '%s %s' % (self.asTables, sColumn); + self.sColumn = sColumn; ##< Normally only applicable if one table. + self.fExpanded = None; ##< Tristate (None, False, True) + self.oSub = oSub; ##< type: FilterCriterion + + +class ModelFilterBase(ModelBase): + """ + Base class for filters. + + Filters are used to narrow down data that is displayed in a list or + report. This class differs a little from ModelDataBase in that it is not + tied to a database table, but one or more database queries that are + typically rather complicated. + + The filter object has two roles: + + 1. It is used by a ModelLogicBase descendant to store the available + filtering options for data begin displayed. + + 2. It decodes and stores the filtering options submitted by the user so + a ModeLogicBase descendant can use it to construct WHERE statements. + + The ModelFilterBase class is related to the ModelDataBase class in that it + decodes user parameters and stores data, however it is not a descendant. + + Note! In order to reduce URL lengths, we use very very brief parameter + names for the filters. + """ + + def __init__(self): + ModelBase.__init__(self); + self.aCriteria = [] # type: list[FilterCriterion] + + def _initFromParamsWorker(self, oDisp, oCriterion): # (,FilterCriterion) + """ Worker for initFromParams. """ + if oCriterion.sType == FilterCriterion.ksType_UInt: + oCriterion.aoSelected = oDisp.getListOfIntParams(oCriterion.sVarNm, iMin = 0, aiDefaults = []); + elif oCriterion.sType == FilterCriterion.ksType_UIntNil: + oCriterion.aoSelected = oDisp.getListOfIntParams(oCriterion.sVarNm, iMin = -1, aiDefaults = []); + elif oCriterion.sType == FilterCriterion.ksType_String: + oCriterion.aoSelected = oDisp.getListOfStrParams(oCriterion.sVarNm, asDefaults = []); + if len(oCriterion.aoSelected) > 100: + raise TMExceptionBase('Variable %s has %u value, max allowed is 100!' + % (oCriterion.sVarNm, len(oCriterion.aoSelected))); + for sValue in oCriterion.aoSelected: + if len(sValue) > 64 \ + or '\'' in sValue \ + or sValue[-1] == '\\': + raise TMExceptionBase('Variable %s has an illegal value "%s"!' % (oCriterion.sVarNm, sValue)); + elif oCriterion.sType == FilterCriterion.ksType_Ranges: + def convertRangeNumber(sValue): + """ Helper """ + sValue = sValue.strip(); + if sValue and sValue not in ('inf', 'Inf', 'INf', 'INF', 'InF', 'iNf', 'iNF', 'inF',): + try: return int(sValue); + except: pass; + return None; + + for sRange in oDisp.getStringParam(oCriterion.sVarNm, sDefault = '').split(','): + sRange = sRange.strip(); + if sRange and sRange != '-' and any(ch.isdigit() for ch in sRange): + asValues = sRange.split('-'); + if len(asValues) == 1: + asValues = [asValues[0], asValues[0]]; + elif len(asValues) > 2: + asValues = [asValues[0], asValues[-1]]; + tTuple = (convertRangeNumber(asValues[0]), convertRangeNumber(asValues[1])); + if tTuple[0] is not None and tTuple[1] is not None and tTuple[0] > tTuple[1]: + tTuple = (tTuple[1], tTuple[0]); + oCriterion.aoSelected.append(tTuple); + else: + assert False; + if oCriterion.aoSelected: + oCriterion.sState = FilterCriterion.ksState_Selected; + else: + oCriterion.sState = FilterCriterion.ksState_NotSelected; + + if oCriterion.sKind == FilterCriterion.ksKind_ElementOfOrNot: + oCriterion.fInverted = oDisp.getBoolParam(oCriterion.sInvVarNm, fDefault = False); + + if oCriterion.oSub is not None: + self._initFromParamsWorker(oDisp, oCriterion.oSub); + return; + + def initFromParams(self, oDisp): # type: (WuiDispatcherBase) -> self + """ + Initialize the object from parameters. + + Returns self. Raises exception on invalid parameter value. + """ + + for oCriterion in self.aCriteria: + self._initFromParamsWorker(oDisp, oCriterion); + return self; + + def strainParameters(self, dParams, aAdditionalParams = None): + """ Filters just the parameters relevant to this filter, returning a copy. """ + + # Collect the parameter names. + dWanted = {}; + for oCrit in self.aCriteria: + dWanted[oCrit.sVarNm] = 1; + if oCrit.sInvVarNm: + dWanted[oCrit.sInvVarNm] = 1; + + # Add additional stuff. + if aAdditionalParams: + for sParam in aAdditionalParams: + dWanted[sParam] = 1; + + # To the straining. + dRet = {}; + for sKey in dParams: + if sKey in dWanted: + dRet[sKey] = dParams[sKey]; + return dRet; + + +class ModelLogicBase(ModelBase): # pylint: disable=too-few-public-methods + """ + Something all classes in the logic classes the logical model inherits from. + """ + + def __init__(self, oDb): + ModelBase.__init__(self); + + # + # Note! Do not create a connection here if None, we need to DB share + # connection with all other logic objects so we can perform half + # complex transactions involving several logic objects. + # + self._oDb = oDb; + + def getDbConnection(self): + """ + Gets the database connection. + This should only be used for instantiating other ModelLogicBase children. + """ + return self._oDb; + + def _dbRowsToModelDataList(self, oModelDataType, aaoRows = None): + """ + Helper for conerting a simple fetch into a list of ModelDataType python objects. + + If aaoRows is None, we'll fetchAll from the database ourselves. + + The oModelDataType must be a class derived from ModelDataBase and implement + the initFormDbRow method. + + Returns a list of oModelDataType instances. + """ + assert issubclass(oModelDataType, ModelDataBase); + aoRet = []; + if aaoRows is None: + aaoRows = self._oDb.fetchAll(); + for aoRow in aaoRows: + aoRet.append(oModelDataType().initFromDbRow(aoRow)); + return aoRet; + + + +class AttributeChangeEntry(object): # pylint: disable=too-few-public-methods + """ + Data class representing the changes made to one attribute. + """ + + def __init__(self, sAttr, oNewRaw, oOldRaw, sNewText, sOldText): + self.sAttr = sAttr; + self.oNewRaw = oNewRaw; + self.oOldRaw = oOldRaw; + self.sNewText = sNewText; + self.sOldText = sOldText; + +class AttributeChangeEntryPre(AttributeChangeEntry): # pylint: disable=too-few-public-methods + """ + AttributeChangeEntry for preformatted values. + """ + + def __init__(self, sAttr, oNewRaw, oOldRaw, sNewText, sOldText): + AttributeChangeEntry.__init__(self, sAttr, oNewRaw, oOldRaw, sNewText, sOldText); + +class ChangeLogEntry(object): # pylint: disable=too-few-public-methods + """ + A change log entry returned by the fetchChangeLog method typically + implemented by ModelLogicBase child classes. + """ + + def __init__(self, uidAuthor, sAuthor, tsEffective, tsExpire, oNewRaw, oOldRaw, aoChanges): + self.uidAuthor = uidAuthor; + self.sAuthor = sAuthor; + self.tsEffective = tsEffective; + self.tsExpire = tsExpire; + self.oNewRaw = oNewRaw; + self.oOldRaw = oOldRaw; # Note! NULL for the last entry. + self.aoChanges = aoChanges; + diff --git a/src/VBox/ValidationKit/testmanager/core/build.py b/src/VBox/ValidationKit/testmanager/core/build.py new file mode 100755 index 00000000..bfe50ca5 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/build.py @@ -0,0 +1,891 @@ +# -*- coding: utf-8 -*- +# $Id: build.py $ + +""" +Test Manager - Builds. +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Standard python imports. +import os; +import unittest; + +# Validation Kit imports. +from testmanager import config; +from testmanager.core import coreconsts; +from testmanager.core.base import ModelDataBase, ModelDataBaseTestCase, ModelLogicBase, TMExceptionBase, \ + TMTooManyRows, TMInvalidData, TMRowNotFound, TMRowInUse; + + +class BuildCategoryData(ModelDataBase): + """ + A build category. + """ + + ksIdAttr = 'idBuildCategory'; + + ksParam_idBuildCategory = 'BuildCategory_idBuildCategory'; + ksParam_sProduct = 'BuildCategory_sProduct'; + ksParam_sRepository = 'BuildCategory_sRepository'; + ksParam_sBranch = 'BuildCategory_sBranch'; + ksParam_sType = 'BuildCategory_sType'; + ksParam_asOsArches = 'BuildCategory_asOsArches'; + + kasAllowNullAttributes = ['idBuildCategory', ]; + + def __init__(self): + ModelDataBase.__init__(self); + + # + # Initialize with defaults. + # See the database for explanations of each of these fields. + # + self.idBuildCategory = None; + self.sProduct = None; + self.sRepository = None; + self.sBranch = None; + self.sType = None; + self.asOsArches = None; + + def initFromDbRow(self, aoRow): + """ + Re-initializes the object from a SELECT * FROM BuildCategories row. + Returns self. Raises exception if aoRow is None. + """ + if aoRow is None: + raise TMRowNotFound('BuildCategory not found.'); + + self.idBuildCategory = aoRow[0]; + self.sProduct = aoRow[1]; + self.sRepository = aoRow[2]; + self.sBranch = aoRow[3]; + self.sType = aoRow[4]; + self.asOsArches = sorted(aoRow[5]); + return self; + + def initFromDbWithId(self, oDb, idBuildCategory, tsNow = None, sPeriodBack = None): + """ + Initialize from the database, given the ID of a row. + """ + _ = tsNow; _ = sPeriodBack; # No history in this table. + oDb.execute('SELECT * FROM BuildCategories WHERE idBuildCategory = %s', (idBuildCategory,)); + aoRow = oDb.fetchOne() + if aoRow is None: + raise TMRowNotFound('idBuildCategory=%s not found' % (idBuildCategory, )); + return self.initFromDbRow(aoRow); + + def initFromValues(self, sProduct, sRepository, sBranch, sType, asOsArches, idBuildCategory = None): + """ + Reinitializes form a set of values. + return self. + """ + self.idBuildCategory = idBuildCategory; + self.sProduct = sProduct; + self.sRepository = sRepository; + self.sBranch = sBranch; + self.sType = sType; + self.asOsArches = asOsArches; + return self; + + def _validateAndConvertAttribute(self, sAttr, sParam, oValue, aoNilValues, fAllowNull, oDb): + # Handle sType and asOsArches specially. + if sAttr == 'sType': + (oNewValue, sError) = ModelDataBase._validateAndConvertAttribute(self, sAttr, sParam, oValue, + aoNilValues, fAllowNull, oDb); + if sError is None and self.sType.lower() != self.sType: + sError = 'Invalid build type value'; + + elif sAttr == 'asOsArches': + (oNewValue, sError) = self.validateListOfStr(oValue, aoNilValues = aoNilValues, fAllowNull = fAllowNull, + asValidValues = coreconsts.g_kasOsDotCpusAll); + if sError is not None and oNewValue is not None: + oNewValue = sorted(oNewValue); # Must be sorted! + + else: + return ModelDataBase._validateAndConvertAttribute(self, sAttr, sParam, oValue, aoNilValues, fAllowNull, oDb); + + return (oNewValue, sError); + + def matchesOsArch(self, sOs, sArch): + """ Checks if the build matches the given OS and architecture. """ + if sOs + '.' + sArch in self.asOsArches: + return True; + if sOs + '.noarch' in self.asOsArches: + return True; + if 'os-agnostic.' + sArch in self.asOsArches: + return True; + if 'os-agnostic.noarch' in self.asOsArches: + return True; + return False; + + +class BuildCategoryLogic(ModelLogicBase): # pylint: disable=too-few-public-methods + """ + Build categories database logic. + """ + + def __init__(self, oDb): + ModelLogicBase.__init__(self, oDb) + self.dCache = None; + + def fetchForListing(self, iStart, cMaxRows, tsNow, aiSortColumns = None): + """ + Fetches testboxes for listing. + + Returns an array (list) of UserAccountData items, empty list if none. + Raises exception on error. + """ + _ = tsNow; _ = aiSortColumns; + self._oDb.execute('SELECT *\n' + 'FROM BuildCategories\n' + 'ORDER BY sProduct, sRepository, sBranch, sType, idBuildCategory\n' + 'LIMIT %s OFFSET %s\n' + , (cMaxRows, iStart,)); + + aoRows = []; + for _ in range(self._oDb.getRowCount()): + aoRows.append(BuildCategoryData().initFromDbRow(self._oDb.fetchOne())); + return aoRows; + + def fetchForCombo(self): + """ + Gets the list of Build Categories for a combo box. + Returns an array of (value [idBuildCategory], drop-down-name [info], + hover-text [info]) tuples. + """ + self._oDb.execute('SELECT *\n' + 'FROM BuildCategories\n' + 'ORDER BY sProduct, sBranch, sType, asOsArches') + + aaoRows = self._oDb.fetchAll() + aoRet = [] + for aoRow in aaoRows: + oData = BuildCategoryData().initFromDbRow(aoRow) + + sInfo = '%s / %s / %s / %s' % \ + (oData.sProduct, + oData.sBranch, + oData.sType, + ', '.join(oData.asOsArches)) + + # Make short info string if necessary + sInfo = sInfo if len(sInfo) < 70 else (sInfo[:70] + '...') + + oInfoItem = (oData.idBuildCategory, sInfo, sInfo) + aoRet.append(oInfoItem) + + return aoRet + + def addEntry(self, oData, uidAuthor = None, fCommit = False): + """ + Standard method for adding a build category. + """ + + # Lazy bird warning! Reuse the soft addBuildCategory method. + self.addBuildCategory(oData, fCommit); + _ = uidAuthor; + return True; + + def removeEntry(self, uidAuthor, idBuildCategory, fCascade = False, fCommit = False): + """ + Tries to delete the build category. + Note! Does not implement cascading. This is intentional! + """ + + # + # Check that the build category isn't used by anyone. + # + self._oDb.execute('SELECT COUNT(idBuild)\n' + 'FROM Builds\n' + 'WHERE idBuildCategory = %s\n' + , (idBuildCategory,)); + cBuilds = self._oDb.fetchOne()[0]; + if cBuilds > 0: + raise TMRowInUse('Build category #%d is used by %d builds and can therefore not be deleted.' + % (idBuildCategory, cBuilds,)); + + # + # Ok, it's not used, so just delete it. + # (No history on this table. This code is for typos.) + # + self._oDb.execute('DELETE FROM Builds\n' + 'WHERE idBuildCategory = %s\n' + , (idBuildCategory,)); + + self._oDb.maybeCommit(fCommit); + _ = uidAuthor; _ = fCascade; + return True; + + def cachedLookup(self, idBuildCategory): + """ + Looks up the most recent BuildCategoryData object for idBuildCategory + via an object cache. + + Returns a shared BuildCategoryData object. None if not found. + Raises exception on DB error. + """ + if self.dCache is None: + self.dCache = self._oDb.getCache('BuildCategoryData'); + oEntry = self.dCache.get(idBuildCategory, None); + if oEntry is None: + self._oDb.execute('SELECT *\n' + 'FROM BuildCategories\n' + 'WHERE idBuildCategory = %s\n' + , (idBuildCategory, )); + if self._oDb.getRowCount() == 1: + aaoRow = self._oDb.fetchOne(); + oEntry = BuildCategoryData(); + oEntry.initFromDbRow(aaoRow); + self.dCache[idBuildCategory] = oEntry; + return oEntry; + + # + # Other methods. + # + + def tryFetch(self, idBuildCategory): + """ + Try fetch the build category with the given ID. + Returns BuildCategoryData instance if found, None if not found. + May raise exception on database error. + """ + self._oDb.execute('SELECT *\n' + 'FROM BuildCategories\n' + 'WHERE idBuildCategory = %s\n' + , (idBuildCategory,)) + aaoRows = self._oDb.fetchAll() + if not aaoRows: + return None; + if len(aaoRows) != 1: + raise self._oDb.integrityException('Duplicates in BuildCategories: %s' % (aaoRows,)); + return BuildCategoryData().initFromDbRow(aaoRows[0]) + + def tryFindByData(self, oData): + """ + Tries to find the matching build category from the sProduct, sBranch, + sType and asOsArches members of oData. + + Returns a valid build category ID and an updated oData object if found. + Returns None and unmodified oData object if not found. + May raise exception on database error. + """ + self._oDb.execute('SELECT *\n' + 'FROM BuildCategories\n' + 'WHERE sProduct = %s\n' + ' AND sRepository = %s\n' + ' AND sBranch = %s\n' + ' AND sType = %s\n' + ' AND asOsArches = %s\n' + , ( oData.sProduct, + oData.sRepository, + oData.sBranch, + oData.sType, + sorted(oData.asOsArches), + )); + aaoRows = self._oDb.fetchAll(); + if not aaoRows: + return None; + if len(aaoRows) > 1: + raise self._oDb.integrityException('Duplicates in BuildCategories: %s' % (aaoRows,)); + + oData.initFromDbRow(aaoRows[0]); + return oData.idBuildCategory; + + def addBuildCategory(self, oData, fCommit = False): + """ + Add Build Category record into the database if needed, returning updated oData. + Raises exception on input and database errors. + """ + + # Check BuildCategoryData before do anything + dDataErrors = oData.validateAndConvert(self._oDb, oData.ksValidateFor_Add); + if dDataErrors: + raise TMInvalidData('Invalid data passed to addBuildCategory(): %s' % (dDataErrors,)); + + # Does it already exist? + if self.tryFindByData(oData) is None: + # No, We'll have to add it. + self._oDb.execute('INSERT INTO BuildCategories (sProduct, sRepository, sBranch, sType, asOsArches)\n' + 'VALUES (%s, %s, %s, %s, %s)\n' + 'RETURNING idBuildCategory' + , ( oData.sProduct, + oData.sRepository, + oData.sBranch, + oData.sType, + sorted(oData.asOsArches), + )); + oData.idBuildCategory = self._oDb.fetchOne()[0]; + + self._oDb.maybeCommit(fCommit); + return oData; + + +class BuildData(ModelDataBase): + """ + A build. + """ + + ksIdAttr = 'idBuild'; + + ksParam_idBuild = 'Build_idBuild'; + ksParam_tsCreated = 'Build_tsCreated'; + ksParam_tsEffective = 'Build_tsEffective'; + ksParam_tsExpire = 'Build_tsExpire'; + ksParam_uidAuthor = 'Build_uidAuthor'; + ksParam_idBuildCategory = 'Build_idBuildCategory'; + ksParam_iRevision = 'Build_iRevision'; + ksParam_sVersion = 'Build_sVersion'; + ksParam_sLogUrl = 'Build_sLogUrl'; + ksParam_sBinaries = 'Build_sBinaries'; + ksParam_fBinariesDeleted = 'Build_fBinariesDeleted'; + + kasAllowNullAttributes = ['idBuild', 'tsCreated', 'tsEffective', 'tsExpire', 'uidAuthor', 'tsCreated', 'sLogUrl']; + + + def __init__(self): + ModelDataBase.__init__(self); + + # + # Initialize with defaults. + # See the database for explanations of each of these fields. + # + self.idBuild = None; + self.tsCreated = None; + self.tsEffective = None; + self.tsExpire = None; + self.uidAuthor = None; + self.idBuildCategory = None; + self.iRevision = None; + self.sVersion = None; + self.sLogUrl = None; + self.sBinaries = None; + self.fBinariesDeleted = False; + + def initFromDbRow(self, aoRow): + """ + Re-initializes the object from a SELECT * FROM Builds row. + Returns self. Raises exception if aoRow is None. + """ + if aoRow is None: + raise TMRowNotFound('Build not found.'); + + self.idBuild = aoRow[0]; + self.tsCreated = aoRow[1]; + self.tsEffective = aoRow[2]; + self.tsExpire = aoRow[3]; + self.uidAuthor = aoRow[4]; + self.idBuildCategory = aoRow[5]; + self.iRevision = aoRow[6]; + self.sVersion = aoRow[7]; + self.sLogUrl = aoRow[8]; + self.sBinaries = aoRow[9]; + self.fBinariesDeleted = aoRow[10]; + return self; + + def initFromDbWithId(self, oDb, idBuild, tsNow = None, sPeriodBack = None): + """ + Initialize from the database, given the ID of a row. + """ + oDb.execute(self.formatSimpleNowAndPeriodQuery(oDb, + 'SELECT *\n' + 'FROM Builds\n' + 'WHERE idBuild = %s\n' + , ( idBuild,), tsNow, sPeriodBack)); + aoRow = oDb.fetchOne() + if aoRow is None: + raise TMRowNotFound('idBuild=%s not found (tsNow=%s sPeriodBack=%s)' % (idBuild, tsNow, sPeriodBack,)); + return self.initFromDbRow(aoRow); + + def areFilesStillThere(self): + """ + Try check if the build files are still there. + + Returns True if they are, None if we cannot tell, and False if one or + more are missing. + """ + if self.fBinariesDeleted: + return False; + + for sBinary in self.sBinaries.split(','): + sBinary = sBinary.strip(); + if not sBinary: + continue; + # Same URL tests as in webutils.downloadFile(). + if sBinary.startswith('http://') \ + or sBinary.startswith('https://') \ + or sBinary.startswith('ftp://'): + # URL - don't bother trying to verify that (we don't use it atm). + fRc = None; + else: + # File. + if config.g_ksBuildBinRootDir is not None: + sFullPath = os.path.join(config.g_ksBuildBinRootDir, sBinary); + fRc = os.path.isfile(sFullPath); + if not fRc \ + and not os.path.isfile(os.path.join(config.g_ksBuildBinRootDir, config.g_ksBuildBinRootFile)): + fRc = None; # Root file missing, so the share might not be mounted correctly. + else: + fRc = None; + if fRc is not True: + return fRc; + + return True; + + +class BuildDataEx(BuildData): + """ + Complete data set. + """ + + kasInternalAttributes = [ 'oCat', ]; + + def __init__(self): + BuildData.__init__(self); + self.oCat = None; + + def initFromDbRow(self, aoRow): + """ + Reinitialize from a SELECT Builds.*, BuildCategories.* FROM Builds, BuildCategories query. + Returns self. Raises exception if aoRow is None. + """ + if aoRow is None: + raise TMRowNotFound('Build not found.'); + BuildData.initFromDbRow(self, aoRow); + self.oCat = BuildCategoryData().initFromDbRow(aoRow[11:]); + return self; + + def initFromDbWithId(self, oDb, idBuild, tsNow = None, sPeriodBack = None): + """ + Reinitialize from database given a row ID. + Returns self. Raises exception on database error or if the ID is invalid. + """ + oDb.execute(self.formatSimpleNowAndPeriodQuery(oDb, + 'SELECT Builds.*, BuildCategories.*\n' + 'FROM Builds, BuildCategories\n' + 'WHERE idBuild = %s\n' + ' AND Builds.idBuildCategory = BuildCategories.idBuildCategory\n' + , ( idBuild,), tsNow, sPeriodBack, 'Builds.')); + aoRow = oDb.fetchOne() + if aoRow is None: + raise TMRowNotFound('idBuild=%s not found (tsNow=%s sPeriodBack=%s)' % (idBuild, tsNow, sPeriodBack,)); + return self.initFromDbRow(aoRow); + + def convertFromParamNull(self): + raise TMExceptionBase('Not implemented'); + + def isEqual(self, oOther): + raise TMExceptionBase('Not implemented'); + + + +class BuildLogic(ModelLogicBase): # pylint: disable=too-few-public-methods + """ + Build database logic (covers build categories as well as builds). + """ + + def __init__(self, oDb): + ModelLogicBase.__init__(self, oDb) + self.dCache = None; + + # + # Standard methods. + # + + def fetchForListing(self, iStart, cMaxRows, tsNow, aiSortColumns = None): + """ + Fetches builds for listing. + + Returns an array (list) of BuildDataEx items, empty list if none. + Raises exception on error. + """ + _ = aiSortColumns; + + if tsNow is None: + self._oDb.execute('SELECT *\n' + 'FROM Builds, BuildCategories\n' + 'WHERE Builds.idBuildCategory = BuildCategories.idBuildCategory\n' + ' AND Builds.tsExpire = \'infinity\'::TIMESTAMP\n' + 'ORDER BY tsCreated DESC\n' + 'LIMIT %s OFFSET %s\n' + , (cMaxRows, iStart,)); + else: + self._oDb.execute('SELECT *\n' + 'FROM Builds, BuildCategories\n' + 'WHERE Builds.idBuildCategory = BuildCategories.idBuildCategory\n' + ' AND Builds.tsExpire > %s\n' + ' AND Builds.tsEffective <= %s\n' + 'ORDER BY tsCreated DESC\n' + 'LIMIT %s OFFSET %s\n' + , (tsNow, tsNow, cMaxRows, iStart,)); + + aoRows = []; + for _ in range(self._oDb.getRowCount()): + aoRows.append(BuildDataEx().initFromDbRow(self._oDb.fetchOne())); + return aoRows; + + def addEntry(self, oBuildData, uidAuthor = None, fCommit = False): + """ + Adds the build to the database, optionally adding the build category if + a BuildDataEx object used and it's necessary. + + Returns updated data object. Raises exception on failure. + """ + + # Find/Add the build category if specified. + if isinstance(oBuildData, BuildDataEx) \ + and oBuildData.idBuildCategory is None: + BuildCategoryLogic(self._oDb).addBuildCategory(oBuildData.oCat, fCommit = False); + oBuildData.idBuildCategory = oBuildData.oCat.idBuildCategory; + + # Add the build. + self._oDb.execute('INSERT INTO Builds (uidAuthor,\n' + ' idBuildCategory,\n' + ' iRevision,\n' + ' sVersion,\n' + ' sLogUrl,\n' + ' sBinaries,\n' + ' fBinariesDeleted)\n' + 'VALUES (%s, %s, %s, %s, %s, %s, %s)\n' + 'RETURNING idBuild, tsCreated\n' + , ( uidAuthor, + oBuildData.idBuildCategory, + oBuildData.iRevision, + oBuildData.sVersion, + oBuildData.sLogUrl, + oBuildData.sBinaries, + oBuildData.fBinariesDeleted, + )); + aoRow = self._oDb.fetchOne(); + oBuildData.idBuild = aoRow[0]; + oBuildData.tsCreated = aoRow[1]; + + self._oDb.maybeCommit(fCommit); + return oBuildData; + + def editEntry(self, oData, uidAuthor = None, fCommit = False): + """Modify database record""" + + # + # Validate input and get current data. + # + dErrors = oData.validateAndConvert(self._oDb, oData.ksValidateFor_Edit); + if dErrors: + raise TMInvalidData('editEntry invalid input: %s' % (dErrors,)); + oOldData = BuildData().initFromDbWithId(self._oDb, oData.idBuild); + + # + # Do the work. + # + if not oData.isEqualEx(oOldData, [ 'tsEffective', 'tsExpire', 'uidAuthor' ]): + self._historizeBuild(oData.idBuild); + self._oDb.execute('INSERT INTO Builds (uidAuthor,\n' + ' idBuild,\n' + ' tsCreated,\n' + ' idBuildCategory,\n' + ' iRevision,\n' + ' sVersion,\n' + ' sLogUrl,\n' + ' sBinaries,\n' + ' fBinariesDeleted)\n' + 'VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)\n' + 'RETURNING idBuild, tsCreated\n' + , ( uidAuthor, + oData.idBuild, + oData.tsCreated, + oData.idBuildCategory, + oData.iRevision, + oData.sVersion, + oData.sLogUrl, + oData.sBinaries, + oData.fBinariesDeleted, + )); + + self._oDb.maybeCommit(fCommit); + return True; + + def removeEntry(self, uidAuthor, idBuild, fCascade = False, fCommit = False): + """ + Historize record + """ + + # + # No non-historic refs here, so just go ahead and expire the build. + # + _ = fCascade; + _ = uidAuthor; ## @todo record deleter. + + self._historizeBuild(idBuild, None); + + self._oDb.maybeCommit(fCommit); + return True; + + def cachedLookup(self, idBuild): + """ + Looks up the most recent BuildDataEx object for idBuild + via an object cache. + + Returns a shared BuildDataEx object. None if not found. + Raises exception on DB error. + """ + if self.dCache is None: + self.dCache = self._oDb.getCache('BuildDataEx'); + oEntry = self.dCache.get(idBuild, None); + if oEntry is None: + self._oDb.execute('SELECT Builds.*, BuildCategories.*\n' + 'FROM Builds, BuildCategories\n' + 'WHERE Builds.idBuild = %s\n' + ' AND Builds.idBuildCategory = BuildCategories.idBuildCategory\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , (idBuild, )); + if self._oDb.getRowCount() == 0: + # Maybe it was deleted, try get the last entry. + self._oDb.execute('SELECT Builds.*, BuildCategories.*\n' + 'FROM Builds, BuildCategories\n' + 'WHERE Builds.idBuild = %s\n' + ' AND Builds.idBuildCategory = BuildCategories.idBuildCategory\n' + 'ORDER BY tsExpire DESC\n' + 'LIMIT 1\n' + , (idBuild, )); + elif self._oDb.getRowCount() > 1: + raise self._oDb.integrityException('%s infinity rows for %s' % (self._oDb.getRowCount(), idBuild)); + + if self._oDb.getRowCount() == 1: + aaoRow = self._oDb.fetchOne(); + oEntry = BuildDataEx(); + oEntry.initFromDbRow(aaoRow); + self.dCache[idBuild] = oEntry; + return oEntry; + + + # + # Other methods. + # + + def tryFindSameBuildForOsArch(self, oBuildEx, sOs, sCpuArch): + """ + Attempts to find a matching build for the given OS.ARCH. May return + the input build if if matches. + + Returns BuildDataEx instance if found, None if none. May raise + exception on database error. + """ + + if oBuildEx.oCat.matchesOsArch(sOs, sCpuArch): + return oBuildEx; + + self._oDb.execute('SELECT Builds.*, BuildCategories.*\n' + 'FROM Builds, BuildCategories\n' + 'WHERE BuildCategories.sProduct = %s\n' + ' AND BuildCategories.sBranch = %s\n' + ' AND BuildCategories.sType = %s\n' + ' AND ( %s = ANY(BuildCategories.asOsArches)\n' + ' OR %s = ANY(BuildCategories.asOsArches)\n' + ' OR %s = ANY(BuildCategories.asOsArches))\n' + ' AND Builds.idBuildCategory = BuildCategories.idBuildCategory\n' + ' AND Builds.tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND Builds.iRevision = %s\n' + ' AND Builds.sRelease = %s\n' + ' AND Builds.fBinariesDeleted IS FALSE\n' + 'ORDER BY tsCreated DESC\n' + 'LIMIT 4096\n' # stay sane. + , (oBuildEx.oCat.sProduct, + oBuildEx.oCat.sBranch, + oBuildEx.oCat.sType, + '%s.%s' % (sOs, sCpuArch), + '%s.noarch' % (sOs,), + 'os-agnostic.%s' % (sCpuArch,), + 'os-agnostic.noarch', + oBuildEx.iRevision, + oBuildEx.sRelease, + ) ); + aaoRows = self._oDb.fetchAll(); + + for aoRow in aaoRows: + oBuildExRet = BuildDataEx().initFromDbRow(aoRow); + if not self.isBuildBlacklisted(oBuildExRet): + return oBuildExRet; + + return None; + + def isBuildBlacklisted(self, oBuildEx): + """ + Checks if the given build is blacklisted + Returns True/False. May raise exception on database error. + """ + + asOsAgnosticArch = []; + asOsNoArch = []; + for sOsArch in oBuildEx.oCat.asOsArches: + asParts = sOsArch.split('.'); + if len(asParts) != 2 or not asParts[0] or not asParts[1]: + raise self._oDb.integrityException('Bad build asOsArches value: %s (idBuild=%s idBuildCategory=%s)' + % (sOsArch, oBuildEx.idBuild, oBuildEx.idBuildCategory)); + asOsNoArch.append(asParts[0] + '.noarch'); + asOsNoArch.append('os-agnostic.' + asParts[1]); + + self._oDb.execute('SELECT COUNT(*)\n' + 'FROM BuildBlacklist\n' + 'WHERE BuildBlacklist.tsExpire > CURRENT_TIMESTAMP\n' + ' AND BuildBlacklist.tsEffective <= CURRENT_TIMESTAMP\n' + ' AND BuildBlacklist.sProduct = %s\n' + ' AND BuildBlacklist.sBranch = %s\n' + ' AND ( BuildBlacklist.asTypes is NULL\n' + ' OR %s = ANY(BuildBlacklist.asTypes))\n' + ' AND ( BuildBlacklist.asOsArches is NULL\n' + ' OR %s && BuildBlacklist.asOsArches\n' ## @todo check array rep! Need overload? + ' OR %s && BuildBlacklist.asOsArches\n' + ' OR %s && BuildBlacklist.asOsArches\n' + ' OR %s = ANY(BuildBlacklist.asOsArches))\n' + ' AND BuildBlacklist.iFirstRevision <= %s\n' + ' AND BuildBlacklist.iLastRevision >= %s\n' + , (oBuildEx.oCat.sProduct, + oBuildEx.oCat.sBranch, + oBuildEx.oCat.sType, + oBuildEx.oCat.asOsArches, + asOsAgnosticArch, + asOsNoArch, + 'os-agnostic.noarch', + oBuildEx.iRevision, + oBuildEx.iRevision, + ) ); + return self._oDb.fetchOne()[0] > 0; + + + def getById(self, idBuild): + """ + Get build record by its id + """ + self._oDb.execute('SELECT Builds.*, BuildCategories.*\n' + 'FROM Builds, BuildCategories\n' + 'WHERE Builds.idBuild=%s\n' + ' AND Builds.idBuildCategory=BuildCategories.idBuildCategory\n' + ' AND Builds.tsExpire = \'infinity\'::TIMESTAMP\n', (idBuild,)) + + aRows = self._oDb.fetchAll() + if len(aRows) not in (0, 1): + raise TMTooManyRows('Found more than one build with the same credentials. Database structure is corrupted.') + try: + return BuildDataEx().initFromDbRow(aRows[0]) + except IndexError: + return None + + + def getAll(self, tsEffective = None): + """ + Gets the list of all builds. + Returns an array of BuildDataEx instances. + """ + if tsEffective is None: + self._oDb.execute('SELECT Builds.*, BuildCategories.*\n' + 'FROM Builds, BuildCategories\n' + 'WHERE Builds.tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND Builds.idBuildCategory=BuildCategories.idBuildCategory') + else: + self._oDb.execute('SELECT Builds.*, BuildCategories.*\n' + 'FROM Builds, BuildCategories\n' + 'WHERE Builds.tsExpire > %s\n' + ' AND Builds.tsEffective <= %s' + ' AND Builds.idBuildCategory=BuildCategories.idBuildCategory' + , (tsEffective, tsEffective)) + aoRet = [] + for aoRow in self._oDb.fetchAll(): + aoRet.append(BuildDataEx().initFromDbRow(aoRow)) + return aoRet + + + def markDeletedByBinaries(self, sBinaries, fCommit = False): + """ + Marks zero or more builds deleted given the build binaries. + + Returns the number of affected builds. + """ + # Fetch a list of affected build IDs (generally 1 build), and used the + # editEntry method to do the rest. This isn't 100% optimal, but it's + # short and simple, the main effort is anyway the first query. + self._oDb.execute('SELECT idBuild\n' + 'FROM Builds\n' + 'WHERE sBinaries = %s\n' + ' AND fBinariesDeleted = FALSE\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , (sBinaries,)); + aaoRows = self._oDb.fetchAll(); + for aoRow in aaoRows: + oData = BuildData().initFromDbWithId(self._oDb, aoRow[0]); + assert not oData.fBinariesDeleted; + oData.fBinariesDeleted = True; + self.editEntry(oData, fCommit = False); + self._oDb.maybeCommit(fCommit); + return len(aaoRows); + + + + # + # Internal helpers. + # + + def _historizeBuild(self, idBuild, tsExpire = None): + """ Historizes the current entry for the specified build. """ + if tsExpire is None: + self._oDb.execute('UPDATE Builds\n' + 'SET tsExpire = CURRENT_TIMESTAMP\n' + 'WHERE idBuild = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , (idBuild,)); + else: + self._oDb.execute('UPDATE Builds\n' + 'SET tsExpire = %s\n' + 'WHERE idBuild = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , (tsExpire, idBuild,)); + return True; + +# +# Unit testing. +# + +# pylint: disable=missing-docstring +class BuildCategoryDataTestCase(ModelDataBaseTestCase): + def setUp(self): + self.aoSamples = [BuildCategoryData(),]; + +class BuildDataTestCase(ModelDataBaseTestCase): + def setUp(self): + self.aoSamples = [BuildData(),]; + +if __name__ == '__main__': + unittest.main(); + # not reached. + diff --git a/src/VBox/ValidationKit/testmanager/core/buildblacklist.py b/src/VBox/ValidationKit/testmanager/core/buildblacklist.py new file mode 100755 index 00000000..487fb9f0 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/buildblacklist.py @@ -0,0 +1,324 @@ +# -*- coding: utf-8 -*- +# $Id: buildblacklist.py $ + +""" +Test Manager - Builds Blacklist. +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Validation Kit imports. +from testmanager.core.base import ModelDataBase, ModelLogicBase, TMInvalidData, TMRowNotFound; + + +class BuildBlacklistData(ModelDataBase): + """ + Build Blacklist Data. + """ + + ksIdAttr = 'idBlacklisting'; + + ksParam_idBlacklisting = 'BuildBlacklist_idBlacklisting' + ksParam_tsEffective = 'BuildBlacklist_tsEffective' + ksParam_tsExpire = 'BuildBlacklist_tsExpire' + ksParam_uidAuthor = 'BuildBlacklist_uidAuthor' + ksParam_idFailureReason = 'BuildBlacklist_idFailureReason' + ksParam_sProduct = 'BuildBlacklist_sProduct' + ksParam_sBranch = 'BuildBlacklist_sBranch' + ksParam_asTypes = 'BuildBlacklist_asTypes' + ksParam_asOsArches = 'BuildBlacklist_asOsArches' + ksParam_iFirstRevision = 'BuildBlacklist_iFirstRevision' + ksParam_iLastRevision = 'BuildBlacklist_iLastRevision' + + kasAllowNullAttributes = [ 'idBlacklisting', + 'tsEffective', + 'tsExpire', + 'uidAuthor', + 'asTypes', + 'asOsArches' ]; + + def __init__(self): + ModelDataBase.__init__(self); + + # + # Initialize with defaults. + # See the database for explanations of each of these fields. + # + self.idBlacklisting = None + self.tsEffective = None + self.tsExpire = None + self.uidAuthor = None + self.idFailureReason = None + self.sProduct = None + self.sBranch = None + self.asTypes = None + self.asOsArches = None + self.iFirstRevision = None + self.iLastRevision = None + + def initFromDbRow(self, aoRow): + """ + Re-initializes the data with a row from a SELECT * FROM BuildBlacklist. + + Returns self. Raises exception if the row is None or otherwise invalid. + """ + + if aoRow is None: + raise TMRowNotFound('Build Blacklist item not found.') + + self.idBlacklisting = aoRow[0] + self.tsEffective = aoRow[1] + self.tsExpire = aoRow[2] + self.uidAuthor = aoRow[3] + self.idFailureReason = aoRow[4] + self.sProduct = aoRow[5] + self.sBranch = aoRow[6] + self.asTypes = aoRow[7] + self.asOsArches = aoRow[8] + self.iFirstRevision = aoRow[9] + self.iLastRevision = aoRow[10] + + return self; + + def initFromDbWithId(self, oDb, idBlacklisting, tsNow = None, sPeriodBack = None): + """ + Initialize the object from the database. + """ + oDb.execute(self.formatSimpleNowAndPeriodQuery(oDb, + 'SELECT *\n' + 'FROM BuildBlacklist\n' + 'WHERE idBlacklisting = %s\n' + , ( idBlacklisting,), tsNow, sPeriodBack)); + aoRow = oDb.fetchOne() + if aoRow is None: + raise TMRowNotFound('idBlacklisting=%s not found (tsNow=%s sPeriodBack=%s)' + % (idBlacklisting, tsNow, sPeriodBack,)); + return self.initFromDbRow(aoRow); + + +class BuildBlacklistLogic(ModelLogicBase): # pylint: disable=too-few-public-methods + """ + Build Back List logic. + """ + + def __init__(self, oDb): + ModelLogicBase.__init__(self, oDb) + self.dCache = None; + + def fetchForListing(self, iStart, cMaxRows, tsNow, aiSortColumns = None): + """ + Fetches Build Blacklist records. + + Returns an array (list) of BuildBlacklistData items, empty list if none. + Raises exception on error. + """ + _ = aiSortColumns; + + if tsNow is None: + self._oDb.execute('SELECT *\n' + 'FROM BuildBlacklist\n' + 'WHERE tsExpire = \'infinity\'::TIMESTAMP\n' + 'ORDER BY idBlacklisting DESC\n' + 'LIMIT %s OFFSET %s\n' + , (cMaxRows, iStart,)); + else: + self._oDb.execute('SELECT *\n' + 'FROM BuildBlacklist\n' + 'WHERE tsExpire > %s\n' + ' AND tsEffective <= %s\n' + 'ORDER BY idBlacklisting DESC\n' + 'LIMIT %s OFFSET %s\n' + , (tsNow, tsNow, cMaxRows, iStart,)); + + aoRows = [] + for aoRow in self._oDb.fetchAll(): + aoRows.append(BuildBlacklistData().initFromDbRow(aoRow)) + return aoRows + + def addEntry(self, oData, uidAuthor, fCommit = False): + """ + Adds a blacklisting to the database. + """ + self._oDb.execute('INSERT INTO BuildBlacklist (\n' + ' uidAuthor,\n' + ' idFailureReason,\n' + ' sProduct,\n' + ' sBranch,\n' + ' asTypes,\n' + ' asOsArches,\n' + ' iFirstRevision,\n' + ' iLastRevision)\n' + 'VALUES (%s, %s, %s, %s, %s, %s, %s, %s)' + , ( uidAuthor, + oData.idFailureReason, + oData.sProduct, + oData.sBranch, + oData.asTypes, + oData.asOsArches, + oData.iFirstRevision, + oData.iLastRevision,) ); + self._oDb.maybeCommit(fCommit); + return True + + def editEntry(self, oData, uidAuthor, fCommit = False): + """ + Modifies a blacklisting. + """ + + # + # Validate inputs and read in the old(/current) data. + # + assert isinstance(oData, BuildBlacklistData); + dErrors = oData.validateAndConvert(self._oDb, oData.ksValidateFor_Edit); + if dErrors: + raise TMInvalidData('editEntry invalid input: %s' % (dErrors,)); + + oOldData = BuildBlacklistData().initFromDbWithId(self._oDb, oData.idBlacklisting); + + # + # Update the data that needs updating. + # + if not oData.isEqualEx(oOldData, [ 'tsEffective', 'tsExpire', 'uidAuthor', ]): + self._historizeEntry(oData.idBlacklisting, None); + self._readdEntry(uidAuthor, oData, None); + self._oDb.maybeCommit(fCommit); + return True; + + + def removeEntry(self, uidAuthor, idBlacklisting, fCascade = False, fCommit = False): + """ + Deletes a test group. + """ + _ = fCascade; # Not applicable. + + oData = BuildBlacklistData().initFromDbWithId(self._oDb, idBlacklisting); + + (tsCur, tsCurMinusOne) = self._oDb.getCurrentTimestamps(); + if oData.tsEffective not in (tsCur, tsCurMinusOne): + self._historizeEntry(idBlacklisting, tsCurMinusOne); + self._readdEntry(uidAuthor, oData, tsCurMinusOne); + self._historizeEntry(idBlacklisting); + self._oDb.execute('UPDATE BuildBlacklist\n' + 'SET tsExpire = CURRENT_TIMESTAMP\n' + 'WHERE idBlacklisting = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , (idBlacklisting,)); + self._oDb.maybeCommit(fCommit); + return True; + + + def cachedLookup(self, idBlacklisting): + """ + Looks up the most recent BuildBlacklistData object for idBlacklisting + via an object cache. + + Returns a shared BuildBlacklistData object. None if not found. + Raises exception on DB error. + """ + if self.dCache is None: + self.dCache = self._oDb.getCache('BuildBlacklistData'); + oEntry = self.dCache.get(idBlacklisting, None); + if oEntry is None: + self._oDb.execute('SELECT *\n' + 'FROM BuildBlacklist\n' + 'WHERE idBlacklisting = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , (idBlacklisting, )); + if self._oDb.getRowCount() == 0: + # Maybe it was deleted, try get the last entry. + self._oDb.execute('SELECT *\n' + 'FROM BuildBlacklist\n' + 'WHERE idBlacklisting = %s\n' + 'ORDER BY tsExpire DESC\n' + 'LIMIT 1\n' + , (idBlacklisting, )); + elif self._oDb.getRowCount() > 1: + raise self._oDb.integrityException('%s infinity rows for %s' % (self._oDb.getRowCount(), idBlacklisting)); + + if self._oDb.getRowCount() == 1: + aaoRow = self._oDb.fetchOne(); + oEntry = BuildBlacklistData(); + oEntry.initFromDbRow(aaoRow); + self.dCache[idBlacklisting] = oEntry; + return oEntry; + + + # + # Helpers. + # + + def _historizeEntry(self, idBlacklisting, tsExpire = None): + """ + Historizes the current entry for the given backlisting. + """ + if tsExpire is None: + tsExpire = self._oDb.getCurrentTimestamp(); + self._oDb.execute('UPDATE BuildBlacklist\n' + 'SET tsExpire = %s\n' + 'WHERE idBlacklisting = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , ( tsExpire, idBlacklisting, )); + return True; + + def _readdEntry(self, uidAuthor, oData, tsEffective = None): + """ + Re-adds the BuildBlacklist entry. Used by editEntry and removeEntry. + """ + if tsEffective is None: + tsEffective = self._oDb.getCurrentTimestamp(); + self._oDb.execute('INSERT INTO BuildBlacklist (\n' + ' uidAuthor,\n' + ' tsEffective,\n' + ' idBlacklisting,\n' + ' idFailureReason,\n' + ' sProduct,\n' + ' sBranch,\n' + ' asTypes,\n' + ' asOsArches,\n' + ' iFirstRevision,\n' + ' iLastRevision)\n' + 'VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)\n' + , ( uidAuthor, + tsEffective, + oData.idBlacklisting, + oData.idFailureReason, + oData.sProduct, + oData.sBranch, + oData.asTypes, + oData.asOsArches, + oData.iFirstRevision, + oData.iLastRevision,) ); + return True; + diff --git a/src/VBox/ValidationKit/testmanager/core/buildsource.py b/src/VBox/ValidationKit/testmanager/core/buildsource.py new file mode 100755 index 00000000..782b5ee0 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/buildsource.py @@ -0,0 +1,524 @@ +# -*- coding: utf-8 -*- +# $Id: buildsource.py $ + +""" +Test Manager - Build Sources. +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Standard python imports. +import unittest; + +# Validation Kit imports. +from common import utils; +from testmanager.core.base import ModelDataBase, ModelDataBaseTestCase, ModelLogicBase, TMRowAlreadyExists, \ + TMRowInUse, TMInvalidData, TMRowNotFound; +from testmanager.core import coreconsts; + + +class BuildSourceData(ModelDataBase): + """ + A build source. + """ + + ksIdAttr = 'idBuildSrc'; + + ksParam_idBuildSrc = 'BuildSource_idBuildSrc'; + ksParam_tsEffective = 'BuildSource_tsEffective'; + ksParam_tsExpire = 'BuildSource_tsExpire'; + ksParam_uidAuthor = 'BuildSource_uidAuthor'; + ksParam_sName = 'BuildSource_sName'; + ksParam_sDescription = 'BuildSource_sDescription'; + ksParam_sProduct = 'BuildSource_sProduct'; + ksParam_sBranch = 'BuildSource_sBranch'; + ksParam_asTypes = 'BuildSource_asTypes'; + ksParam_asOsArches = 'BuildSource_asOsArches'; + ksParam_iFirstRevision = 'BuildSource_iFirstRevision'; + ksParam_iLastRevision = 'BuildSource_iLastRevision'; + ksParam_cSecMaxAge = 'BuildSource_cSecMaxAge'; + + kasAllowNullAttributes = [ 'idBuildSrc', 'tsEffective', 'tsExpire', 'uidAuthor', 'sDescription', 'asTypes', + 'asOsArches', 'iFirstRevision', 'iLastRevision', 'cSecMaxAge' ]; + + def __init__(self): + ModelDataBase.__init__(self); + + # + # Initialize with defaults. + # See the database for explanations of each of these fields. + # + self.idBuildSrc = None; + self.tsEffective = None; + self.tsExpire = None; + self.uidAuthor = None; + self.sName = None; + self.sDescription = None; + self.sProduct = None; + self.sBranch = None; + self.asTypes = None; + self.asOsArches = None; + self.iFirstRevision = None; + self.iLastRevision = None; + self.cSecMaxAge = None; + + def initFromDbRow(self, aoRow): + """ + Re-initializes the object from a SELECT * FROM BuildSources row. + Returns self. Raises exception if aoRow is None. + """ + if aoRow is None: + raise TMRowNotFound('Build source not found.'); + + self.idBuildSrc = aoRow[0]; + self.tsEffective = aoRow[1]; + self.tsExpire = aoRow[2]; + self.uidAuthor = aoRow[3]; + self.sName = aoRow[4]; + self.sDescription = aoRow[5]; + self.sProduct = aoRow[6]; + self.sBranch = aoRow[7]; + self.asTypes = aoRow[8]; + self.asOsArches = aoRow[9]; + self.iFirstRevision = aoRow[10]; + self.iLastRevision = aoRow[11]; + self.cSecMaxAge = aoRow[12]; + return self; + + def initFromDbWithId(self, oDb, idBuildSrc, tsNow = None, sPeriodBack = None): + """ + Initialize from the database, given the ID of a row. + """ + oDb.execute(self.formatSimpleNowAndPeriodQuery(oDb, + 'SELECT *\n' + 'FROM BuildSources\n' + 'WHERE idBuildSrc = %s\n' + , ( idBuildSrc,), tsNow, sPeriodBack)); + aoRow = oDb.fetchOne() + if aoRow is None: + raise TMRowNotFound('idBuildSrc=%s not found (tsNow=%s sPeriodBack=%s)' % (idBuildSrc, tsNow, sPeriodBack,)); + return self.initFromDbRow(aoRow); + + def _validateAndConvertAttribute(self, sAttr, sParam, oValue, aoNilValues, fAllowNull, oDb): + # Handle asType and asOsArches specially. + if sAttr == 'sType': + (oNewValue, sError) = ModelDataBase._validateAndConvertAttribute(self, sAttr, sParam, oValue, + aoNilValues, fAllowNull, oDb); + if sError is None: + if not self.asTypes: + oNewValue = None; + else: + for sType in oNewValue: + if len(sType) < 2 or sType.lower() != sType: + if sError is None: sError = ''; + else: sError += ', '; + sError += 'invalid value "%s"' % (sType,); + + elif sAttr == 'asOsArches': + (oNewValue, sError) = self.validateListOfStr(oValue, aoNilValues = aoNilValues, fAllowNull = fAllowNull, + asValidValues = coreconsts.g_kasOsDotCpusAll); + if sError is not None and oNewValue is not None: + oNewValue = sorted(oNewValue); # Must be sorted! + + elif sAttr == 'cSecMaxAge' and oValue not in aoNilValues: # Allow human readable interval formats. + (oNewValue, sError) = utils.parseIntervalSeconds(oValue); + else: + return ModelDataBase._validateAndConvertAttribute(self, sAttr, sParam, oValue, aoNilValues, fAllowNull, oDb); + + return (oNewValue, sError); + +class BuildSourceLogic(ModelLogicBase): # pylint: disable=too-few-public-methods + """ + Build source database logic. + """ + + def __init__(self, oDb): + ModelLogicBase.__init__(self, oDb) + self.dCache = None; + + # + # Standard methods. + # + + def fetchForListing(self, iStart, cMaxRows, tsNow, aiSortColumns = None): + """ + Fetches build sources. + + Returns an array (list) of BuildSourceData items, empty list if none. + Raises exception on error. + """ + _ = aiSortColumns; + + if tsNow is None: + self._oDb.execute('SELECT *\n' + 'FROM BuildSources\n' + 'WHERE tsExpire = \'infinity\'::TIMESTAMP\n' + 'ORDER BY idBuildSrc DESC\n' + 'LIMIT %s OFFSET %s\n' + , (cMaxRows, iStart,)); + else: + self._oDb.execute('SELECT *\n' + 'FROM BuildSources\n' + 'WHERE tsExpire > %s\n' + ' AND tsEffective <= %s\n' + 'ORDER BY idBuildSrc DESC\n' + 'LIMIT %s OFFSET %s\n' + , (tsNow, tsNow, cMaxRows, iStart,)); + + aoRows = [] + for aoRow in self._oDb.fetchAll(): + aoRows.append(BuildSourceData().initFromDbRow(aoRow)) + return aoRows + + def fetchForCombo(self): + """Fetch data which is aimed to be passed to HTML form""" + self._oDb.execute('SELECT idBuildSrc, sName, sProduct\n' + 'FROM BuildSources\n' + 'WHERE tsExpire = \'infinity\'::TIMESTAMP\n' + 'ORDER BY idBuildSrc DESC\n') + asRet = self._oDb.fetchAll(); + asRet.insert(0, (-1, 'None', 'None')); + return asRet; + + + def addEntry(self, oData, uidAuthor, fCommit = False): + """ + Add a new build source to the database. + """ + + # + # Validate the input. + # + dErrors = oData.validateAndConvert(self._oDb, oData.ksValidateFor_Add); + if dErrors: + raise TMInvalidData('addEntry invalid input: %s' % (dErrors,)); + self._assertUnique(oData, None); + + # + # Add it. + # + self._oDb.execute('INSERT INTO BuildSources (\n' + ' uidAuthor,\n' + ' sName,\n' + ' sDescription,\n' + ' sProduct,\n' + ' sBranch,\n' + ' asTypes,\n' + ' asOsArches,\n' + ' iFirstRevision,\n' + ' iLastRevision,\n' + ' cSecMaxAge)\n' + 'VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)\n' + , ( uidAuthor, + oData.sName, + oData.sDescription, + oData.sProduct, + oData.sBranch, + oData.asTypes, + oData.asOsArches, + oData.iFirstRevision, + oData.iLastRevision, + oData.cSecMaxAge, )); + + self._oDb.maybeCommit(fCommit); + return True; + + def editEntry(self, oData, uidAuthor, fCommit = False): + """ + Modifies a build source. + """ + + # + # Validate the input and read the old entry. + # + dErrors = oData.validateAndConvert(self._oDb, oData.ksValidateFor_Edit); + if dErrors: + raise TMInvalidData('addEntry invalid input: %s' % (dErrors,)); + self._assertUnique(oData, oData.idBuildSrc); + oOldData = BuildSourceData().initFromDbWithId(self._oDb, oData.idBuildSrc); + + # + # Make the changes (if something actually changed). + # + if not oData.isEqualEx(oOldData, [ 'tsEffective', 'tsExpire', 'uidAuthor', ]): + self._historizeBuildSource(oData.idBuildSrc); + self._oDb.execute('INSERT INTO BuildSources (\n' + ' uidAuthor,\n' + ' idBuildSrc,\n' + ' sName,\n' + ' sDescription,\n' + ' sProduct,\n' + ' sBranch,\n' + ' asTypes,\n' + ' asOsArches,\n' + ' iFirstRevision,\n' + ' iLastRevision,\n' + ' cSecMaxAge)\n' + 'VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)\n' + , ( uidAuthor, + oData.idBuildSrc, + oData.sName, + oData.sDescription, + oData.sProduct, + oData.sBranch, + oData.asTypes, + oData.asOsArches, + oData.iFirstRevision, + oData.iLastRevision, + oData.cSecMaxAge, )); + self._oDb.maybeCommit(fCommit); + return True; + + def removeEntry(self, uidAuthor, idBuildSrc, fCascade = False, fCommit = False): + """ + Deletes a build sources. + """ + + # + # Check cascading. + # + if fCascade is not True: + self._oDb.execute('SELECT idSchedGroup, sName\n' + 'FROM SchedGroups\n' + 'WHERE idBuildSrc = %s\n' + ' OR idBuildSrcTestSuite = %s\n' + , (idBuildSrc, idBuildSrc,)); + if self._oDb.getRowCount() > 0: + asGroups = []; + for aoRow in self._oDb.fetchAll(): + asGroups.append('%s (#%d)' % (aoRow[1], aoRow[0])); + raise TMRowInUse('Build source #%d is used by one or more scheduling groups: %s' + % (idBuildSrc, ', '.join(asGroups),)); + else: + self._oDb.execute('UPDATE SchedGroups\n' + 'SET idBuildSrc = NULL\n' + 'WHERE idBuildSrc = %s' + , ( idBuildSrc,)); + self._oDb.execute('UPDATE SchedGroups\n' + 'SET idBuildSrcTestSuite = NULL\n' + 'WHERE idBuildSrcTestSuite = %s' + , ( idBuildSrc,)); + + # + # Do the job. + # + self._historizeBuildSource(idBuildSrc, None); + _ = uidAuthor; ## @todo record deleter. + + self._oDb.maybeCommit(fCommit); + return True; + + def cachedLookup(self, idBuildSrc): + """ + Looks up the most recent BuildSourceData object for idBuildSrc + via an object cache. + + Returns a shared BuildSourceData object. None if not found. + Raises exception on DB error. + """ + if self.dCache is None: + self.dCache = self._oDb.getCache('BuildSourceData'); + oEntry = self.dCache.get(idBuildSrc, None); + if oEntry is None: + self._oDb.execute('SELECT *\n' + 'FROM BuildSources\n' + 'WHERE idBuildSrc = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , (idBuildSrc, )); + if self._oDb.getRowCount() == 0: + # Maybe it was deleted, try get the last entry. + self._oDb.execute('SELECT *\n' + 'FROM BuildSources\n' + 'WHERE idBuildSrc = %s\n' + 'ORDER BY tsExpire DESC\n' + 'LIMIT 1\n' + , (idBuildSrc, )); + elif self._oDb.getRowCount() > 1: + raise self._oDb.integrityException('%s infinity rows for %s' % (self._oDb.getRowCount(), idBuildSrc)); + + if self._oDb.getRowCount() == 1: + aaoRow = self._oDb.fetchOne(); + oEntry = BuildSourceData(); + oEntry.initFromDbRow(aaoRow); + self.dCache[idBuildSrc] = oEntry; + return oEntry; + + # + # Other methods. + # + + def openBuildCursor(self, oBuildSource, sOs, sCpuArch, tsNow): + """ + Opens a cursor (SELECT) using the criteria found in the build source + and the given OS.CPUARCH. + + Returns database cursor. May raise exception on bad input or logic error. + + Used by SchedulerBase. + """ + + oCursor = self._oDb.openCursor(); + + # + # Construct the extra conditionals. + # + sExtraConditions = ''; + + # Types + if oBuildSource.asTypes is not None and oBuildSource.asTypes: + if len(oBuildSource.asTypes) == 1: + sExtraConditions += oCursor.formatBindArgs(' AND BuildCategories.sType = %s', (oBuildSource.asTypes[0],)); + else: + sExtraConditions += oCursor.formatBindArgs(' AND BuildCategories.sType IN (%s', (oBuildSource.asTypes[0],)) + for i in range(1, len(oBuildSource.asTypes) - 1): + sExtraConditions += oCursor.formatBindArgs(', %s', (oBuildSource.asTypes[i],)); + sExtraConditions += oCursor.formatBindArgs(', %s)\n', (oBuildSource.asTypes[-1],)); + + # BuildSource OSes.ARCHes. (Paranoia: use a dictionary to avoid duplicate values.) + if oBuildSource.asOsArches is not None and oBuildSource.asOsArches: + sExtraConditions += oCursor.formatBindArgs(' AND BuildCategories.asOsArches && %s', (oBuildSource.asOsArches,)); + + # TestBox OSes.ARCHes. (Paranoia: use a dictionary to avoid duplicate values.) + dOsDotArches = {}; + dOsDotArches[sOs + '.' + sCpuArch] = 1; + dOsDotArches[sOs + '.' + coreconsts.g_ksCpuArchAgnostic] = 1; + dOsDotArches[coreconsts.g_ksOsAgnostic + '.' + sCpuArch] = 1; + dOsDotArches[coreconsts.g_ksOsDotArchAgnostic] = 1; + sExtraConditions += oCursor.formatBindArgs(' AND BuildCategories.asOsArches && %s', (list(dOsDotArches.keys()),)); + + # Revision range. + if oBuildSource.iFirstRevision is not None: + sExtraConditions += oCursor.formatBindArgs(' AND Builds.iRevision >= %s\n', (oBuildSource.iFirstRevision,)); + if oBuildSource.iLastRevision is not None: + sExtraConditions += oCursor.formatBindArgs(' AND Builds.iRevision <= %s\n', (oBuildSource.iLastRevision,)); + + # Max age. + if oBuildSource.cSecMaxAge is not None: + sExtraConditions += oCursor.formatBindArgs(' AND Builds.tsCreated >= (%s - \'%s seconds\'::INTERVAL)\n', + (tsNow, oBuildSource.cSecMaxAge,)); + + # + # Execute the query. + # + oCursor.execute('SELECT Builds.*, BuildCategories.*,\n' + ' EXISTS( SELECT tsExpire\n' + ' FROM BuildBlacklist\n' + ' WHERE BuildBlacklist.tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND BuildBlacklist.sProduct = %s\n' + ' AND BuildBlacklist.sBranch = %s\n' + ' AND BuildBlacklist.iFirstRevision <= Builds.iRevision\n' + ' AND BuildBlacklist.iLastRevision >= Builds.iRevision ) AS fMaybeBlacklisted\n' + 'FROM Builds, BuildCategories\n' + 'WHERE Builds.idBuildCategory = BuildCategories.idBuildCategory\n' + ' AND Builds.tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND Builds.tsEffective <= %s\n' + ' AND Builds.fBinariesDeleted is FALSE\n' + ' AND BuildCategories.sProduct = %s\n' + ' AND BuildCategories.sBranch = %s\n' + + sExtraConditions + + 'ORDER BY Builds.idBuild DESC\n' + 'LIMIT 256\n' + , ( oBuildSource.sProduct, oBuildSource.sBranch, + tsNow, oBuildSource.sProduct, oBuildSource.sBranch,)); + + return oCursor; + + + def getById(self, idBuildSrc): + """Get Build Source data by idBuildSrc""" + + self._oDb.execute('SELECT *\n' + 'FROM BuildSources\n' + 'WHERE tsExpire = \'infinity\'::timestamp\n' + ' AND idBuildSrc = %s;', (idBuildSrc,)) + aRows = self._oDb.fetchAll() + if len(aRows) not in (0, 1): + raise self._oDb.integrityException( + 'Found more than one build sources with the same credentials. Database structure is corrupted.') + try: + return BuildSourceData().initFromDbRow(aRows[0]) + except IndexError: + return None + + # + # Internal helpers. + # + + def _assertUnique(self, oData, idBuildSrcIgnore): + """ Checks that the build source name is unique, raises exception if it isn't. """ + self._oDb.execute('SELECT idBuildSrc\n' + 'FROM BuildSources\n' + 'WHERE sName = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + + ('' if idBuildSrcIgnore is None else ' AND idBuildSrc <> %d\n' % (idBuildSrcIgnore,)) + , ( oData.sName, )) + if self._oDb.getRowCount() > 0: + raise TMRowAlreadyExists('A build source with name "%s" already exist.' % (oData.sName,)); + return True; + + + def _historizeBuildSource(self, idBuildSrc, tsExpire = None): + """ Historizes the current build source entry. """ + if tsExpire is None: + self._oDb.execute('UPDATE BuildSources\n' + 'SET tsExpire = CURRENT_TIMESTAMP\n' + 'WHERE idBuildSrc = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , ( idBuildSrc, )); + else: + self._oDb.execute('UPDATE BuildSources\n' + 'SET tsExpire = %s\n' + 'WHERE idBuildSrc = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , ( tsExpire, idBuildSrc, )); + return True; + + + + + +# +# Unit testing. +# + +# pylint: disable=missing-docstring +class BuildSourceDataTestCase(ModelDataBaseTestCase): + def setUp(self): + self.aoSamples = [BuildSourceData(),]; + +if __name__ == '__main__': + unittest.main(); + # not reached. + diff --git a/src/VBox/ValidationKit/testmanager/core/coreconsts.py b/src/VBox/ValidationKit/testmanager/core/coreconsts.py new file mode 100644 index 00000000..bbdc7712 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/coreconsts.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +# $Id: coreconsts.py $ + +""" +Test Manager - Test Manager Constants (without a more appropriate home). +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + +## OS agnostic. +g_ksOsAgnostic = 'os-agnostic'; +## All known OSes, except the agnostic one. +# See KBUILD_OSES in kBuild/header.kmk for reference. +g_kasOses = ['darwin', 'dos', 'dragonfly', 'freebsd', 'haiku', 'l4', 'linux', 'netbsd', 'nt', 'openbsd', 'os2', + 'solaris', 'win']; +## All known OSes, including the agnostic one. +# See KBUILD_OSES in kBuild/header.kmk for reference. +g_kasOsesAll = g_kasOses + [g_ksOsAgnostic,]; + + +## Architecture agnostic. +g_ksCpuArchAgnostic = 'noarch'; +## All known CPU architectures, except the agnostic one. +# See KBUILD_ARCHES in kBuild/header.kmk for reference. +g_kasCpuArches = ['amd64', 'x86', 'sparc32', 'sparc64', 's390', 's390x', 'ppc32', 'ppc64', 'mips32', 'mips64', 'ia64', + 'hppa32', 'hppa64', 'arm', 'alpha']; +## All known CPU architectures, except the agnostic one. +# See KBUILD_ARCHES in kBuild/header.kmk for reference. +g_kasCpuArchesAll = g_kasCpuArches + [g_ksCpuArchAgnostic,]; + +## All known build types +# See KBUILD_TYPE in kBuild/header.kmk for reference. +# @note 'blessed' is a special type used for release builds that has been notarized +# or attestation signed by the OS vendor. +g_kasBuildTypesAll = [ 'release', 'strict', 'profile', 'debug', 'asan', 'blessed' ]; + +## OS and CPU architecture agnostic. +g_ksOsDotArchAgnostic = 'os-agnostic.noarch'; +## Combinations of all OSes and CPU architectures, except the two agnostic ones. +# We do some of them by hand to avoid offering too many choices. +g_kasOsDotCpus = \ +[ + 'darwin.amd64', 'darwin.x86', 'darwin.ppc32', 'darwin.ppc64', 'darwin.arm', + 'dos.x86', + 'dragonfly.amd64', 'dragonfly.x86', + 'freebsd.amd64', 'freebsd.x86', 'freebsd.sparc64', 'freebsd.ia64', 'freebsd.ppc32', 'freebsd.ppc64', 'freebsd.arm', + 'freebsd.mips32', 'freebsd.mips64', + 'haiku.amd64', 'haiku.x86', + 'l4.amd64', 'l4.x86', 'l4.ppc32', 'l4.ppc64', 'l4.arm', + 'nt.amd64', 'nt.x86', 'nt.arm', 'nt.ia64', 'nt.mips32', 'nt.ppc32', 'nt.alpha', + 'win.amd64', 'win.x86', 'win.arm', 'win.ia64', 'win.mips32', 'win.ppc32', 'win.alpha', + 'os2.x86', + 'solaris.amd64', 'solaris.x86', 'solaris.sparc32', 'solaris.sparc64', +]; +for sOs in g_kasOses: + if sOs not in ['darwin', 'dos', 'dragonfly', 'freebsd', 'haiku', 'l4', 'nt', 'win', 'os2', 'solaris']: + for sArch in g_kasCpuArches: + g_kasOsDotCpus.append(sOs + '.' + sArch); +g_kasOsDotCpus.sort(); + +## Combinations of all OSes and CPU architectures, including the two agnostic ones. +g_kasOsDotCpusAll = [g_ksOsDotArchAgnostic] +g_kasOsDotCpusAll.extend(g_kasOsDotCpus); +for sOs in g_kasOsesAll: + g_kasOsDotCpusAll.append(sOs + '.' + g_ksCpuArchAgnostic); +for sArch in g_kasCpuArchesAll: + g_kasOsDotCpusAll.append(g_ksOsAgnostic + '.' + sArch); +g_kasOsDotCpusAll.sort(); + diff --git a/src/VBox/ValidationKit/testmanager/core/db.py b/src/VBox/ValidationKit/testmanager/core/db.py new file mode 100755 index 00000000..915336c3 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/db.py @@ -0,0 +1,745 @@ +# -*- coding: utf-8 -*- +# $Id: db.py $ + +""" +Test Manager - Database Interface. +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Standard python imports. +import datetime; +import os; +import sys; +import psycopg2; # pylint: disable=import-error +import psycopg2.extensions; # pylint: disable=import-error + +# Validation Kit imports. +from common import utils, webutils; +from testmanager import config; + +# Fix psycho unicode handling in psycopg2 with python 2.x. +if sys.version_info[0] < 3: + psycopg2.extensions.register_type(psycopg2.extensions.UNICODE); + psycopg2.extensions.register_type(psycopg2.extensions.UNICODEARRAY); +else: + unicode = str; # pylint: disable=redefined-builtin,invalid-name + + + +def isDbTimestampInfinity(tsValue): + """ + Checks if tsValue is an infinity timestamp. + """ + ## @todo improve this test... + return tsValue.year >= 9999; + +def isDbTimestamp(oValue): + """ + Checks if oValue is a DB timestamp object. + """ + if isinstance(oValue, datetime.datetime): + return True; + if utils.isString(oValue): + ## @todo detect strings as well. + return False; + return getattr(oValue, 'pydatetime', None) is not None; + +def dbTimestampToDatetime(oValue): + """ + Converts a database timestamp to a datetime instance. + """ + if isinstance(oValue, datetime.datetime): + return oValue; + if utils.isString(oValue): + return utils.parseIsoTimestamp(oValue); + return oValue.pydatetime(); + +def dbTimestampToZuluDatetime(oValue): + """ + Converts a database timestamp to a zulu datetime instance. + """ + tsValue = dbTimestampToDatetime(oValue); + + class UTC(datetime.tzinfo): + """UTC TZ Info Class""" + def utcoffset(self, _): + return datetime.timedelta(0); + def tzname(self, _): + return "UTC"; + def dst(self, _): + return datetime.timedelta(0); + if tsValue.tzinfo is not None: + tsValue = tsValue.astimezone(UTC()); + else: + tsValue = tsValue.replace(tzinfo=UTC()); + return tsValue; + +def dbTimestampPythonNow(): + """ + Gets the current python timestamp in a database compatible way. + """ + return dbTimestampToZuluDatetime(datetime.datetime.utcnow()); + +def dbOneTickIntervalString(): + """ + Returns the interval string for one tick. + + Mogrify the return value into the SQL: + "... %s::INTERVAL ..." + or + "INTERVAL %s" + The completed SQL will contain the necessary ticks. + """ + return '1 microsecond'; + +def dbTimestampMinusOneTick(oValue): + """ + Returns a new timestamp that's one tick before the given one. + """ + oValue = dbTimestampToZuluDatetime(oValue); + return oValue - datetime.timedelta(microseconds = 1); + +def dbTimestampPlusOneTick(oValue): + """ + Returns a new timestamp that's one tick after the given one. + """ + oValue = dbTimestampToZuluDatetime(oValue); + return oValue + datetime.timedelta(microseconds = 1); + +def isDbInterval(oValue): + """ + Checks if oValue is a DB interval object. + """ + if isinstance(oValue, datetime.timedelta): + return True; + return False; + + +class TMDatabaseIntegrityException(Exception): + """ + Herolds a database integrity error up the callstack. + + Do NOT use directly, only thru TMDatabaseConnection.integrityException. + Otherwise, we won't be able to log the issue. + """ + pass; # pylint: disable=unnecessary-pass + + +class TMDatabaseCursor(object): + """ Cursor wrapper class. """ + + def __init__(self, oDb, oCursor): + self._oDb = oDb; + self._oCursor = oCursor; + + def execute(self, sOperation, aoArgs = None): + """ See TMDatabaseConnection.execute()""" + return self._oDb.executeInternal(self._oCursor, sOperation, aoArgs, utils.getCallerName()); + + def callProc(self, sProcedure, aoArgs = None): + """ See TMDatabaseConnection.callProc()""" + return self._oDb.callProcInternal(self._oCursor, sProcedure, aoArgs, utils.getCallerName()); + + def insertList(self, sInsertSql, aoList, fnEntryFmt): + """ See TMDatabaseConnection.insertList. """ + return self._oDb.insertListInternal(self._oCursor, sInsertSql, aoList, fnEntryFmt, utils.getCallerName()); + + def fetchOne(self): + """Wrapper around Psycopg2.cursor.fetchone.""" + return self._oCursor.fetchone(); + + def fetchMany(self, cRows = None): + """Wrapper around Psycopg2.cursor.fetchmany.""" + return self._oCursor.fetchmany(cRows if cRows is not None else self._oCursor.arraysize); + + def fetchAll(self): + """Wrapper around Psycopg2.cursor.fetchall.""" + return self._oCursor.fetchall(); + + def getRowCount(self): + """Wrapper around Psycopg2.cursor.rowcount.""" + return self._oCursor.rowcount; + + def formatBindArgs(self, sStatement, aoArgs): + """Wrapper around Psycopg2.cursor.mogrify.""" + oRet = self._oCursor.mogrify(sStatement, aoArgs); + if sys.version_info[0] >= 3 and not isinstance(oRet, str): + oRet = oRet.decode('utf-8'); + return oRet; + + def copyExpert(self, sSqlCopyStmt, oFile, cbBuf = 8192): + """ See TMDatabaseConnection.copyExpert()""" + return self._oCursor.copy_expert(sSqlCopyStmt, oFile, cbBuf); + + @staticmethod + def isTsInfinity(tsValue): + """ Checks if tsValue is an infinity timestamp. """ + return isDbTimestampInfinity(tsValue); + + +class TMDatabaseConnection(object): + """ + Test Manager Database Access class. + + This class contains no logic, just raw access abstraction and utilities, + as well as some debug help and some statistics. + """ + + def __init__(self, fnDPrint = None, oSrvGlue = None): + """ + Database connection wrapper. + The fnDPrint is for debug logging of all database activity. + + Raises an exception on failure. + """ + + sAppName = '%s-%s' % (os.getpid(), os.path.basename(sys.argv[0]),) + if len(sAppName) >= 64: + sAppName = sAppName[:64]; + os.environ['PGAPPNAME'] = sAppName; + + dArgs = \ + { \ + 'database': config.g_ksDatabaseName, + 'user': config.g_ksDatabaseUser, + 'password': config.g_ksDatabasePassword, + # 'application_name': sAppName, - Darn stale debian! :/ + }; + if config.g_ksDatabaseAddress is not None: + dArgs['host'] = config.g_ksDatabaseAddress; + if config.g_ksDatabasePort is not None: + dArgs['port'] = config.g_ksDatabasePort; + self._oConn = psycopg2.connect(**dArgs); # pylint: disable=star-args + self._oConn.set_client_encoding('UTF-8'); + self._oCursor = self._oConn.cursor(); + self._oExplainConn = None; + self._oExplainCursor = None; + if config.g_kfWebUiSqlTraceExplain and config.g_kfWebUiSqlTrace: + self._oExplainConn = psycopg2.connect(**dArgs); # pylint: disable=star-args + self._oExplainConn.set_client_encoding('UTF-8'); + self._oExplainCursor = self._oExplainConn.cursor(); + self._fTransaction = False; + self._tsCurrent = None; + self._tsCurrentMinusOne = None; + + assert self.isAutoCommitting() is False; + + # Debug and introspection. + self._fnDPrint = fnDPrint; + self._aoTraceBack = []; + + # Exception class handles. + self.oXcptError = psycopg2.Error; + + if oSrvGlue is not None: + oSrvGlue.registerDebugInfoCallback(self.debugInfoCallback); + + # Object caches (used by database logic classes). + self.ddCaches = {}; + + def isAutoCommitting(self): + """ Work around missing autocommit attribute in older versions.""" + return getattr(self._oConn, 'autocommit', False); + + def close(self): + """ + Closes the connection and renders all cursors useless. + """ + if self._oCursor is not None: + self._oCursor.close(); + self._oCursor = None; + + if self._oConn is not None: + self._oConn.close(); + self._oConn = None; + + if self._oExplainCursor is not None: + self._oExplainCursor.close(); + self._oExplainCursor = None; + + if self._oExplainConn is not None: + self._oExplainConn.close(); + self._oExplainConn = None; + + + def _startedTransaction(self): + """ + Called to work the _fTransaction and related variables when starting + a transaction. + """ + self._fTransaction = True; + self._tsCurrent = None; + self._tsCurrentMinusOne = None; + return None; + + def _endedTransaction(self): + """ + Called to work the _fTransaction and related variables when ending + a transaction. + """ + self._fTransaction = False; + self._tsCurrent = None; + self._tsCurrentMinusOne = None; + return None; + + def begin(self): + """ + Currently just for marking where a transaction starts in the code. + """ + assert self._oConn is not None; + assert self.isAutoCommitting() is False; + self._aoTraceBack.append([utils.timestampNano(), 'START TRANSACTION', 0, 0, utils.getCallerName(), None]); + self._startedTransaction(); + return True; + + def commit(self, sCallerName = None): + """ Wrapper around Psycopg2.connection.commit.""" + assert self._fTransaction is True; + + nsStart = utils.timestampNano(); + oRc = self._oConn.commit(); + cNsElapsed = utils.timestampNano() - nsStart; + + if sCallerName is None: + sCallerName = utils.getCallerName(); + self._aoTraceBack.append([nsStart, 'COMMIT', cNsElapsed, 0, sCallerName, None]); + self._endedTransaction(); + return oRc; + + def maybeCommit(self, fCommit): + """ + Commits if fCommit is True. + Returns True if committed, False if not. + """ + if fCommit is True: + self.commit(utils.getCallerName()); + return True; + return False; + + def rollback(self): + """ Wrapper around Psycopg2.connection.rollback.""" + nsStart = utils.timestampNano(); + oRc = self._oConn.rollback(); + cNsElapsed = utils.timestampNano() - nsStart; + + self._aoTraceBack.append([nsStart, 'ROLLBACK', cNsElapsed, 0, utils.getCallerName(), None]); + self._endedTransaction(); + return oRc; + + # + # Internal cursor workers. + # + + def executeInternal(self, oCursor, sOperation, aoArgs, sCallerName): + """ + Execute a query or command. + + Mostly a wrapper around the psycopg2 cursor method with the same name, + but collect data for traceback. + """ + if aoArgs is not None: + sBound = oCursor.mogrify(unicode(sOperation), aoArgs); + elif sOperation.find('%') < 0: + sBound = oCursor.mogrify(unicode(sOperation), []); + else: + sBound = unicode(sOperation); + + if sys.version_info[0] >= 3 and not isinstance(sBound, str): + sBound = sBound.decode('utf-8'); # pylint: disable=redefined-variable-type + + aasExplain = None; + if self._oExplainCursor is not None and not sBound.startswith('DROP'): + try: + if config.g_kfWebUiSqlTraceExplainTiming: + self._oExplainCursor.execute('EXPLAIN (ANALYZE, BUFFERS, COSTS, VERBOSE, TIMING) ' + sBound); + else: + self._oExplainCursor.execute('EXPLAIN (ANALYZE, BUFFERS, COSTS, VERBOSE) ' + sBound); + except Exception as oXcpt: + aasExplain = [ ['Explain exception: '], [str(oXcpt)]]; + try: self._oExplainConn.rollback(); + except: pass; + else: + aasExplain = self._oExplainCursor.fetchall(); + + nsStart = utils.timestampNano(); + try: + oRc = oCursor.execute(sBound); + except Exception as oXcpt: + cNsElapsed = utils.timestampNano() - nsStart; + self._aoTraceBack.append([nsStart, 'oXcpt=%s; Statement: %s' % (oXcpt, sBound), cNsElapsed, 0, sCallerName, None]); + if self._fnDPrint is not None: + self._fnDPrint('db::execute %u ns, caller %s: oXcpt=%s; Statement: %s' + % (cNsElapsed, sCallerName, oXcpt, sBound)); + raise; + cNsElapsed = utils.timestampNano() - nsStart; + + if self._fTransaction is False and not self.isAutoCommitting(): # Even SELECTs starts transactions with psycopg2, see FAQ. + self._aoTraceBack.append([nsStart, '[START TRANSACTION]', 0, 0, sCallerName, None]); + self._startedTransaction(); + self._aoTraceBack.append([nsStart, sBound, cNsElapsed, oCursor.rowcount, sCallerName, aasExplain]); + if self._fnDPrint is not None: + self._fnDPrint('db::execute %u ns, caller %s: "\n%s"' % (cNsElapsed, sCallerName, sBound)); + if self.isAutoCommitting(): + self._aoTraceBack.append([nsStart, '[AUTO COMMIT]', 0, 0, sCallerName, None]); + + return oRc; + + def callProcInternal(self, oCursor, sProcedure, aoArgs, sCallerName): + """ + Call a stored procedure. + + Mostly a wrapper around the psycopg2 cursor method 'callproc', but + collect data for traceback. + """ + if aoArgs is None: + aoArgs = []; + + nsStart = utils.timestampNano(); + try: + oRc = oCursor.callproc(sProcedure, aoArgs); + except Exception as oXcpt: + cNsElapsed = utils.timestampNano() - nsStart; + self._aoTraceBack.append([nsStart, 'oXcpt=%s; Calling: %s(%s)' % (oXcpt, sProcedure, aoArgs), + cNsElapsed, 0, sCallerName, None]); + if self._fnDPrint is not None: + self._fnDPrint('db::callproc %u ns, caller %s: oXcpt=%s; Calling: %s(%s)' + % (cNsElapsed, sCallerName, oXcpt, sProcedure, aoArgs)); + raise; + cNsElapsed = utils.timestampNano() - nsStart; + + if self._fTransaction is False and not self.isAutoCommitting(): # Even SELECTs starts transactions with psycopg2, see FAQ. + self._aoTraceBack.append([nsStart, '[START TRANSACTION]', 0, 0, sCallerName, None]); + self._startedTransaction(); + self._aoTraceBack.append([nsStart, '%s(%s)' % (sProcedure, aoArgs), cNsElapsed, oCursor.rowcount, sCallerName, None]); + if self._fnDPrint is not None: + self._fnDPrint('db::callproc %u ns, caller %s: "%s(%s)"' % (cNsElapsed, sCallerName, sProcedure, aoArgs)); + if self.isAutoCommitting(): + self._aoTraceBack.append([nsStart, '[AUTO COMMIT]', 0, 0, sCallerName, sCallerName, None]); + + return oRc; + + def insertListInternal(self, oCursor, sInsertSql, aoList, fnEntryFmt, sCallerName): + """ + Optimizes the insertion of a list of values. + """ + oRc = None; + asValues = []; + for aoEntry in aoList: + asValues.append(fnEntryFmt(aoEntry)); + if len(asValues) > 256: + oRc = self.executeInternal(oCursor, sInsertSql + 'VALUES' + ', '.join(asValues), None, sCallerName); + asValues = []; + if asValues: + oRc = self.executeInternal(oCursor, sInsertSql + 'VALUES' + ', '.join(asValues), None, sCallerName); + return oRc + + def _fetchOne(self, oCursor): + """Wrapper around Psycopg2.cursor.fetchone.""" + oRow = oCursor.fetchone() + if self._fnDPrint is not None: + self._fnDPrint('db:fetchOne returns: %s' % (oRow,)); + return oRow; + + def _fetchMany(self, oCursor, cRows): + """Wrapper around Psycopg2.cursor.fetchmany.""" + return oCursor.fetchmany(cRows if cRows is not None else oCursor.arraysize); + + def _fetchAll(self, oCursor): + """Wrapper around Psycopg2.cursor.fetchall.""" + return oCursor.fetchall() + + def _getRowCountWorker(self, oCursor): + """Wrapper around Psycopg2.cursor.rowcount.""" + return oCursor.rowcount; + + + # + # Default cursor access. + # + + def execute(self, sOperation, aoArgs = None): + """ + Execute a query or command. + + Mostly a wrapper around the psycopg2 cursor method with the same name, + but collect data for traceback. + """ + return self.executeInternal(self._oCursor, sOperation, aoArgs, utils.getCallerName()); + + def callProc(self, sProcedure, aoArgs = None): + """ + Call a stored procedure. + + Mostly a wrapper around the psycopg2 cursor method 'callproc', but + collect data for traceback. + """ + return self.callProcInternal(self._oCursor, sProcedure, aoArgs, utils.getCallerName()); + + def insertList(self, sInsertSql, aoList, fnEntryFmt): + """ + Optimizes the insertion of a list of values. + """ + return self.insertListInternal(self._oCursor, sInsertSql, aoList, fnEntryFmt, utils.getCallerName()); + + def fetchOne(self): + """Wrapper around Psycopg2.cursor.fetchone.""" + return self._oCursor.fetchone(); + + def fetchMany(self, cRows = None): + """Wrapper around Psycopg2.cursor.fetchmany.""" + return self._oCursor.fetchmany(cRows if cRows is not None else self._oCursor.arraysize); + + def fetchAll(self): + """Wrapper around Psycopg2.cursor.fetchall.""" + return self._oCursor.fetchall(); + + def getRowCount(self): + """Wrapper around Psycopg2.cursor.rowcount.""" + return self._oCursor.rowcount; + + def formatBindArgs(self, sStatement, aoArgs): + """Wrapper around Psycopg2.cursor.mogrify.""" + oRet = self._oCursor.mogrify(sStatement, aoArgs); + if sys.version_info[0] >= 3 and not isinstance(oRet, str): + oRet = oRet.decode('utf-8'); + return oRet; + + def copyExpert(self, sSqlCopyStmt, oFile, cbBuf = 8192): + """ Wrapper around Psycopg2.cursor.copy_expert. """ + return self._oCursor.copy_expert(sSqlCopyStmt, oFile, cbBuf); + + def getCurrentTimestamps(self): + """ + Returns the current timestamp and the current timestamp minus one tick. + This will start a transaction if necessary. + """ + if self._tsCurrent is None: + self.execute('SELECT CURRENT_TIMESTAMP, CURRENT_TIMESTAMP - INTERVAL \'1 microsecond\''); + (self._tsCurrent, self._tsCurrentMinusOne) = self.fetchOne(); + return (self._tsCurrent, self._tsCurrentMinusOne); + + def getCurrentTimestamp(self): + """ + Returns the current timestamp. + This will start a transaction if necessary. + """ + if self._tsCurrent is None: + self.getCurrentTimestamps(); + return self._tsCurrent; + + def getCurrentTimestampMinusOne(self): + """ + Returns the current timestamp minus one tick. + This will start a transaction if necessary. + """ + if self._tsCurrentMinusOne is None: + self.getCurrentTimestamps(); + return self._tsCurrentMinusOne; + + + # + # Additional cursors. + # + def openCursor(self): + """ + Opens a new cursor (TMDatabaseCursor). + """ + oCursor = self._oConn.cursor(); + return TMDatabaseCursor(self, oCursor); + + # + # Cache support. + # + def getCache(self, sType): + """ Returns the cache dictionary for this data type. """ + dRet = self.ddCaches.get(sType, None); + if dRet is None: + dRet = {}; + self.ddCaches[sType] = dRet; + return dRet; + + + # + # Utilities. + # + + @staticmethod + def isTsInfinity(tsValue): + """ Checks if tsValue is an infinity timestamp. """ + return isDbTimestampInfinity(tsValue); + + # + # Error stuff. + # + def integrityException(self, sMessage): + """ + Database integrity reporter and exception factory. + Returns an TMDatabaseIntegrityException which the caller can raise. + """ + ## @todo Create a new database connection and log the issue in the SystemLog table. + ## Alternatively, rollback whatever is going on and do it using the current one. + return TMDatabaseIntegrityException(sMessage); + + + # + # Debugging. + # + + def dprint(self, sText): + """ + Debug output. + """ + if not self._fnDPrint: + return False; + self._fnDPrint(sText); + return True; + + def debugHtmlReport(self, tsStart = 0): + """ + Used to get a SQL activity dump as HTML, usually for WuiBase._sDebug. + """ + cNsElapsed = 0; + for aEntry in self._aoTraceBack: + cNsElapsed += aEntry[2]; + + sDebug = '<h3>SQL Debug Log (total time %s ns):</h3>\n' \ + '<table class="tmsqltable">\n' \ + ' <tr>\n' \ + ' <th>No.</th>\n' \ + ' <th>Timestamp (ns)</th>\n' \ + ' <th>Elapsed (ns)</th>\n' \ + ' <th>Rows Returned</th>\n' \ + ' <th>Command</th>\n' \ + ' <th>Caller</th>\n' \ + ' </tr>\n' \ + % (utils.formatNumber(cNsElapsed, ' '),); + + iEntry = 0; + for aEntry in self._aoTraceBack: + iEntry += 1; + sDebug += ' <tr>\n' \ + ' <td>%s</td>\n' \ + ' <td>%s</td>\n' \ + ' <td>%s</td>\n' \ + ' <td>%s</td>\n' \ + ' <td><pre>%s</pre></td>\n' \ + ' <td>%s</td>\n' \ + ' </tr>\n' \ + % (iEntry, + utils.formatNumber(aEntry[0] - tsStart, ' '), + utils.formatNumber(aEntry[2], ' '), + utils.formatNumber(aEntry[3], ' '), + webutils.escapeElem(aEntry[1]), + webutils.escapeElem(aEntry[4]), + ); + if aEntry[5] is not None: + sDebug += ' <tr>\n' \ + ' <td colspan="6"><pre style="white-space: pre-wrap;">%s</pre></td>\n' \ + ' </tr>\n' \ + % (webutils.escapeElem('\n'.join([aoRow[0] for aoRow in aEntry[5]])),); + + sDebug += '</table>'; + return sDebug; + + def debugTextReport(self, tsStart = 0): + """ + Used to get a SQL activity dump as text. + """ + cNsElapsed = 0; + for aEntry in self._aoTraceBack: + cNsElapsed += aEntry[2]; + + sHdr = 'SQL Debug Log (total time %s ns)' % (utils.formatNumber(cNsElapsed),); + sDebug = sHdr + '\n' + '-' * len(sHdr) + '\n'; + + iEntry = 0; + for aEntry in self._aoTraceBack: + iEntry += 1; + sHdr = 'Query #%s Timestamp: %s ns Elapsed: %s ns Rows: %s Caller: %s' \ + % ( iEntry, + utils.formatNumber(aEntry[0] - tsStart), + utils.formatNumber(aEntry[2]), + utils.formatNumber(aEntry[3]), + aEntry[4], ); + sDebug += '\n' + sHdr + '\n' + '-' * len(sHdr) + '\n'; + + sDebug += aEntry[1]; + if sDebug[-1] != '\n': + sDebug += '\n'; + + if aEntry[5] is not None: + sDebug += 'Explain:\n' \ + ' %s\n' \ + % ( '\n'.join([aoRow[0] for aoRow in aEntry[5]]),); + + return sDebug; + + def debugInfoCallback(self, oGlue, fHtml): + """ Called back by the glue code on error. """ + oGlue.write('\n'); + if not fHtml: oGlue.write(self.debugTextReport()); + else: oGlue.write(self.debugHtmlReport()); + oGlue.write('\n'); + return True; + + def debugEnableExplain(self): + """ Enabled explain. """ + if self._oExplainConn is None: + dArgs = \ + { \ + 'database': config.g_ksDatabaseName, + 'user': config.g_ksDatabaseUser, + 'password': config.g_ksDatabasePassword, + # 'application_name': sAppName, - Darn stale debian! :/ + }; + if config.g_ksDatabaseAddress is not None: + dArgs['host'] = config.g_ksDatabaseAddress; + if config.g_ksDatabasePort is not None: + dArgs['port'] = config.g_ksDatabasePort; + self._oExplainConn = psycopg2.connect(**dArgs); # pylint: disable=star-args + self._oExplainCursor = self._oExplainConn.cursor(); + return True; + + def debugDisableExplain(self): + """ Disables explain. """ + self._oExplainCursor = None; + self._oExplainConn = None + return True; + + def debugIsExplainEnabled(self): + """ Check if explaining of SQL statements is enabled. """ + return self._oExplainConn is not None; + diff --git a/src/VBox/ValidationKit/testmanager/core/dbobjcache.py b/src/VBox/ValidationKit/testmanager/core/dbobjcache.py new file mode 100755 index 00000000..b059dccd --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/dbobjcache.py @@ -0,0 +1,200 @@ +# -*- coding: utf-8 -*- +# $Id: dbobjcache.py $ + +""" +Test Manager - Database object cache. +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Validation Kit imports. +from testmanager.core.base import ModelLogicBase; + + +class DatabaseObjCache(ModelLogicBase): + """ + Database object cache. + + This is mainly for reports and test results where we wish to get further + information on a data series or similar. The cache should reduce database + lookups as well as pyhon memory footprint. + + Note! Dependecies are imported when needed to avoid potential cylic dependency issues. + """ + + ## @name Cache object types. + ## @{ + ksObjType_TestResultStrTab_idStrName = 0; + ksObjType_BuildCategory_idBuildCategory = 1; + ksObjType_TestBox_idTestBox = 2; + ksObjType_TestBox_idGenTestBox = 3; + ksObjType_TestCase_idTestCase = 4; + ksObjType_TestCase_idGenTestCase = 5; + ksObjType_TestCaseArgs_idTestCaseArgs = 6; + ksObjType_TestCaseArgs_idGenTestCaseArgs = 7; + ksObjType_VcsRevision_sRepository_iRevision = 8; + ksObjType_End = 9; + ## @} + + def __init__(self, oDb, tsNow = None, sPeriodBack = None, cHoursBack = None): + ModelLogicBase.__init__(self, oDb); + + self.tsNow = tsNow; + self.sPeriodBack = sPeriodBack; + if sPeriodBack is None and cHoursBack is not None: + self.sPeriodBack = '%u hours' % cHoursBack; + + self._adCache = ( + {}, {}, {}, {}, + {}, {}, {}, {}, + {}, + ); + assert(len(self._adCache) == self.ksObjType_End); + + def _handleDbException(self): + """ Deals with database exceptions. """ + #self._oDb.rollback(); + return False; + + def getTestResultString(self, idStrName): + """ Gets a string from the TestResultStrTab. """ + sRet = self._adCache[self.ksObjType_TestResultStrTab_idStrName].get(idStrName); + if sRet is None: + # Load cache entry. + self._oDb.execute('SELECT sValue FROM TestResultStrTab WHERE idStr = %s', (idStrName,)); + sRet = self._oDb.fetchOne()[0]; + self._adCache[self.ksObjType_TestResultStrTab_idStrName][idStrName] = sRet + return sRet; + + def getBuildCategory(self, idBuildCategory): + """ Gets the corresponding BuildCategoryData object. """ + oRet = self._adCache[self.ksObjType_BuildCategory_idBuildCategory].get(idBuildCategory); + if oRet is None: + # Load cache entry. + from testmanager.core.build import BuildCategoryData; + oRet = BuildCategoryData(); + try: oRet.initFromDbWithId(self._oDb, idBuildCategory); + except: self._handleDbException(); raise; + self._adCache[self.ksObjType_BuildCategory_idBuildCategory][idBuildCategory] = oRet; + return oRet; + + def getTestBox(self, idTestBox): + """ Gets the corresponding TestBoxData object. """ + oRet = self._adCache[self.ksObjType_TestBox_idTestBox].get(idTestBox); + if oRet is None: + # Load cache entry. + from testmanager.core.testbox import TestBoxData; + oRet = TestBoxData(); + try: oRet.initFromDbWithId(self._oDb, idTestBox, self.tsNow, self.sPeriodBack); + except: self._handleDbException(); raise; + else: self._adCache[self.ksObjType_TestBox_idGenTestBox][oRet.idGenTestBox] = oRet; + self._adCache[self.ksObjType_TestBox_idTestBox][idTestBox] = oRet; + return oRet; + + def getTestCase(self, idTestCase): + """ Gets the corresponding TestCaseData object. """ + oRet = self._adCache[self.ksObjType_TestCase_idTestCase].get(idTestCase); + if oRet is None: + # Load cache entry. + from testmanager.core.testcase import TestCaseData; + oRet = TestCaseData(); + try: oRet.initFromDbWithId(self._oDb, idTestCase, self.tsNow, self.sPeriodBack); + except: self._handleDbException(); raise; + else: self._adCache[self.ksObjType_TestCase_idGenTestCase][oRet.idGenTestCase] = oRet; + self._adCache[self.ksObjType_TestCase_idTestCase][idTestCase] = oRet; + return oRet; + + def getTestCaseArgs(self, idTestCaseArgs): + """ Gets the corresponding TestCaseArgsData object. """ + oRet = self._adCache[self.ksObjType_TestCaseArgs_idTestCaseArgs].get(idTestCaseArgs); + if oRet is None: + # Load cache entry. + from testmanager.core.testcaseargs import TestCaseArgsData; + oRet = TestCaseArgsData(); + try: oRet.initFromDbWithId(self._oDb, idTestCaseArgs, self.tsNow, self.sPeriodBack); + except: self._handleDbException(); raise; + else: self._adCache[self.ksObjType_TestCaseArgs_idGenTestCaseArgs][oRet.idGenTestCaseArgs] = oRet; + self._adCache[self.ksObjType_TestCaseArgs_idTestCaseArgs][idTestCaseArgs] = oRet; + return oRet; + + def preloadVcsRevInfo(self, sRepository, aiRevisions): + """ + Preloads VCS revision information. + ASSUMES aiRevisions does not contain duplicate keys. + """ + from testmanager.core.vcsrevisions import VcsRevisionData; + dRepo = self._adCache[self.ksObjType_VcsRevision_sRepository_iRevision].get(sRepository); + if dRepo is None: + dRepo = {}; + self._adCache[self.ksObjType_VcsRevision_sRepository_iRevision][sRepository] = dRepo; + aiFiltered = aiRevisions; + else: + aiFiltered = []; + for iRevision in aiRevisions: + if iRevision not in dRepo: + aiFiltered.append(iRevision); + if aiFiltered: + self._oDb.execute('SELECT *\n' + 'FROM VcsRevisions\n' + 'WHERE sRepository = %s\n' + ' AND iRevision IN (' + ','.join([str(i) for i in aiFiltered]) + ')' + , ( sRepository, )); + for aoRow in self._oDb.fetchAll(): + oInfo = VcsRevisionData().initFromDbRow(aoRow); + dRepo[oInfo.iRevision] = oInfo; + return True; + + def getVcsRevInfo(self, sRepository, iRevision): + """ + Gets the corresponding VcsRevisionData object. + May return a default (all NULLs) VcsRevisionData object if the revision + information isn't available in the database yet. + """ + dRepo = self._adCache[self.ksObjType_VcsRevision_sRepository_iRevision].get(sRepository); + if dRepo is not None: + oRet = dRepo.get(iRevision); + else: + dRepo = {}; + self._adCache[self.ksObjType_VcsRevision_sRepository_iRevision][sRepository] = dRepo; + oRet = None; + if oRet is None: + from testmanager.core.vcsrevisions import VcsRevisionLogic; + oRet = VcsRevisionLogic(self._oDb).tryFetch(sRepository, iRevision); + if oRet is None: + from testmanager.core.vcsrevisions import VcsRevisionData; + oRet = VcsRevisionData(); + dRepo[iRevision] = oRet; + return oRet; + diff --git a/src/VBox/ValidationKit/testmanager/core/failurecategory.py b/src/VBox/ValidationKit/testmanager/core/failurecategory.py new file mode 100755 index 00000000..3da0f84b --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/failurecategory.py @@ -0,0 +1,392 @@ +# -*- coding: utf-8 -*- +# $Id: failurecategory.py $ + +""" +Test Manager - Failure Categories. +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Standard Python imports. +import sys; + +# Validation Kit imports. +from testmanager.core.base import ModelDataBase, ModelLogicBase, TMRowInUse, TMInvalidData, TMRowNotFound, \ + ChangeLogEntry, AttributeChangeEntry; +from testmanager.core.useraccount import UserAccountLogic; + +# Python 3 hacks: +if sys.version_info[0] >= 3: + xrange = range; # pylint: disable=redefined-builtin,invalid-name + + +class FailureCategoryData(ModelDataBase): + """ + Failure Category Data. + """ + + ksIdAttr = 'idFailureCategory'; + + ksParam_idFailureCategory = 'FailureCategory_idFailureCategory' + ksParam_tsEffective = 'FailureCategory_tsEffective' + ksParam_tsExpire = 'FailureCategory_tsExpire' + ksParam_uidAuthor = 'FailureCategory_uidAuthor' + ksParam_sShort = 'FailureCategory_sShort' + ksParam_sFull = 'FailureCategory_sFull' + + kasAllowNullAttributes = [ 'idFailureCategory', 'tsEffective', 'tsExpire', 'uidAuthor' ] + + def __init__(self): + ModelDataBase.__init__(self); + + # + # Initialize with defaults. + # See the database for explanations of each of these fields. + # + + self.idFailureCategory = None + self.tsEffective = None + self.tsExpire = None + self.uidAuthor = None + self.sShort = None + self.sFull = None + + def initFromDbRow(self, aoRow): + """ + Re-initializes the data with a row from a SELECT * FROM FailureCategoryes. + + Returns self. Raises exception if the row is None or otherwise invalid. + """ + + if aoRow is None: + raise TMRowNotFound('Failure Category not found.'); + + self.idFailureCategory = aoRow[0] + self.tsEffective = aoRow[1] + self.tsExpire = aoRow[2] + self.uidAuthor = aoRow[3] + self.sShort = aoRow[4] + self.sFull = aoRow[5] + + return self + + def initFromDbWithId(self, oDb, idFailureCategory, tsNow = None, sPeriodBack = None): + """ + Initialize from the database, given the ID of a row. + """ + oDb.execute(self.formatSimpleNowAndPeriodQuery(oDb, + 'SELECT *\n' + 'FROM FailureCategories\n' + 'WHERE idFailureCategory = %s\n' + , ( idFailureCategory,), tsNow, sPeriodBack)); + aoRow = oDb.fetchOne() + if aoRow is None: + raise TMRowNotFound('idFailureCategory=%s not found (tsNow=%s sPeriodBack=%s)' + % (idFailureCategory, tsNow, sPeriodBack,)); + return self.initFromDbRow(aoRow); + + +class FailureCategoryLogic(ModelLogicBase): # pylint: disable=too-few-public-methods + """ + Failure Category logic. + """ + + def __init__(self, oDb): + ModelLogicBase.__init__(self, oDb) + self.dCache = None; + + def fetchForListing(self, iStart, cMaxRows, tsNow, aiSortColumns = None): + """ + Fetches Failure Category records. + + Returns an array (list) of FailureCategoryData items, empty list if none. + Raises exception on error. + """ + _ = aiSortColumns; + + if tsNow is None: + self._oDb.execute('SELECT *\n' + 'FROM FailureCategories\n' + 'WHERE tsExpire = \'infinity\'::TIMESTAMP\n' + 'ORDER BY idFailureCategory ASC\n' + 'LIMIT %s OFFSET %s\n' + , (cMaxRows, iStart,)); + else: + self._oDb.execute('SELECT *\n' + 'FROM FailureCategories\n' + 'WHERE tsExpire > %s\n' + ' AND tsEffective <= %s\n' + 'ORDER BY idFailureCategory ASC\n' + 'LIMIT %s OFFSET %s\n' + , (tsNow, tsNow, cMaxRows, iStart,)); + + aoRows = [] + for aoRow in self._oDb.fetchAll(): + aoRows.append(FailureCategoryData().initFromDbRow(aoRow)) + return aoRows + + + def fetchForChangeLog(self, idFailureCategory, iStart, cMaxRows, tsNow): # pylint: disable=too-many-locals + """ + Fetches change log entries for a failure reason. + + Returns an array of ChangeLogEntry instance and an indicator whether + there are more entries. + Raises exception on error. + """ + if tsNow is None: + tsNow = self._oDb.getCurrentTimestamp(); + + # 1. Get a list of the relevant changes. + self._oDb.execute('SELECT * FROM FailureCategories WHERE idFailureCategory = %s AND tsEffective <= %s\n' + 'ORDER BY tsEffective DESC\n' + 'LIMIT %s OFFSET %s\n' + , ( idFailureCategory, tsNow, cMaxRows + 1, iStart, )); + aoRows = []; + for aoChange in self._oDb.fetchAll(): + aoRows.append(FailureCategoryData().initFromDbRow(aoChange)); + + # 2. Calculate the changes. + aoEntries = []; + for i in xrange(0, len(aoRows) - 1): + oNew = aoRows[i]; + oOld = aoRows[i + 1]; + + aoChanges = []; + for sAttr in oNew.getDataAttributes(): + if sAttr not in [ 'tsEffective', 'tsExpire', 'uidAuthor', ]: + oOldAttr = getattr(oOld, sAttr); + oNewAttr = getattr(oNew, sAttr); + if oOldAttr != oNewAttr: + aoChanges.append(AttributeChangeEntry(sAttr, oNewAttr, oOldAttr, str(oNewAttr), str(oOldAttr))); + + aoEntries.append(ChangeLogEntry(oNew.uidAuthor, None, oNew.tsEffective, oNew.tsExpire, oNew, oOld, aoChanges)); + + # If we're at the end of the log, add the initial entry. + if len(aoRows) <= cMaxRows and aoRows: + oNew = aoRows[-1]; + aoEntries.append(ChangeLogEntry(oNew.uidAuthor, None, oNew.tsEffective, oNew.tsExpire, oNew, None, [])); + + return (UserAccountLogic(self._oDb).resolveChangeLogAuthors(aoEntries), len(aoRows) > cMaxRows); + + + def getFailureCategoriesForCombo(self, tsEffective = None): + """ + Gets the list of Failure Categories for a combo box. + Returns an array of (value [idFailureCategory], drop-down-name [sShort], + hover-text [sFull]) tuples. + """ + if tsEffective is None: + self._oDb.execute('SELECT idFailureCategory, sShort, sFull\n' + 'FROM FailureCategories\n' + 'WHERE tsExpire = \'infinity\'::TIMESTAMP\n' + 'ORDER BY sShort') + else: + self._oDb.execute('SELECT idFailureCategory, sShort, sFull\n' + 'FROM FailureCategories\n' + 'WHERE tsExpire > %s\n' + ' AND tsEffective <= %s\n' + 'ORDER BY sShort' + , (tsEffective, tsEffective)) + return self._oDb.fetchAll() + + + def getById(self, idFailureCategory): + """Get Failure Category data by idFailureCategory""" + + self._oDb.execute('SELECT *\n' + 'FROM FailureCategories\n' + 'WHERE tsExpire = \'infinity\'::timestamp\n' + ' AND idFailureCategory = %s;', (idFailureCategory,)) + aRows = self._oDb.fetchAll() + if len(aRows) not in (0, 1): + raise self._oDb.integrityException( + 'Found more than one failure categories with the same credentials. Database structure is corrupted.') + try: + return FailureCategoryData().initFromDbRow(aRows[0]) + except IndexError: + return None + + + def addEntry(self, oData, uidAuthor, fCommit = False): + """ + Add a failure reason category. + """ + # + # Validate inputs and read in the old(/current) data. + # + assert isinstance(oData, FailureCategoryData); + dErrors = oData.validateAndConvert(self._oDb, oData.ksValidateFor_Add); + if dErrors: + raise TMInvalidData('editEntry invalid input: %s' % (dErrors,)); + + # + # Add the record. + # + self._readdEntry(uidAuthor, oData); + self._oDb.maybeCommit(fCommit); + return True; + + + def editEntry(self, oData, uidAuthor, fCommit = False): + """ + Modifies a failure reason category. + """ + + # + # Validate inputs and read in the old(/current) data. + # + assert isinstance(oData, FailureCategoryData); + dErrors = oData.validateAndConvert(self._oDb, oData.ksValidateFor_Edit); + if dErrors: + raise TMInvalidData('editEntry invalid input: %s' % (dErrors,)); + + oOldData = FailureCategoryData().initFromDbWithId(self._oDb, oData.idFailureCategory); + + # + # Update the data that needs updating. + # + if not oData.isEqualEx(oOldData, [ 'tsEffective', 'tsExpire', 'uidAuthor', ]): + self._historizeEntry(oData.idFailureCategory); + self._readdEntry(uidAuthor, oData); + self._oDb.maybeCommit(fCommit); + return True; + + + def removeEntry(self, uidAuthor, idFailureCategory, fCascade = False, fCommit = False): + """ + Deletes a failure reason category. + """ + _ = fCascade; # too complicated for now. + + # + # Check whether it's being used by other tables and bitch if it is . + # We currently do not implement cascading. + # + self._oDb.execute('SELECT CONCAT(idFailureReason, \' - \', sShort)\n' + 'FROM FailureReasons\n' + 'WHERE idFailureCategory = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , (idFailureCategory,)); + aaoRows = self._oDb.fetchAll(); + if aaoRows: + raise TMRowInUse('Cannot remove failure reason category %u because its being used by: %s' + % (idFailureCategory, ', '.join(aoRow[0] for aoRow in aaoRows),)); + + # + # Do the job. + # + oData = FailureCategoryData().initFromDbWithId(self._oDb, idFailureCategory); + (tsCur, tsCurMinusOne) = self._oDb.getCurrentTimestamps(); + if oData.tsEffective not in (tsCur, tsCurMinusOne): + self._historizeEntry(idFailureCategory, tsCurMinusOne); + self._readdEntry(uidAuthor, oData, tsCurMinusOne); + self._historizeEntry(idFailureCategory); + self._oDb.maybeCommit(fCommit); + return True; + + + def cachedLookup(self, idFailureCategory): + """ + Looks up the most recent FailureCategoryData object for idFailureCategory + via an object cache. + + Returns a shared FailureCategoryData object. None if not found. + Raises exception on DB error. + """ + if self.dCache is None: + self.dCache = self._oDb.getCache('FailureCategory'); + + oEntry = self.dCache.get(idFailureCategory, None); + if oEntry is None: + self._oDb.execute('SELECT *\n' + 'FROM FailureCategories\n' + 'WHERE idFailureCategory = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , (idFailureCategory, )); + if self._oDb.getRowCount() == 0: + # Maybe it was deleted, try get the last entry. + self._oDb.execute('SELECT *\n' + 'FROM FailureCategories\n' + 'WHERE idFailureCategory = %s\n' + 'ORDER BY tsExpire DESC\n' + 'LIMIT 1\n' + , (idFailureCategory, )); + elif self._oDb.getRowCount() > 1: + raise self._oDb.integrityException('%s infinity rows for %s' % (self._oDb.getRowCount(), idFailureCategory)); + + if self._oDb.getRowCount() == 1: + oEntry = FailureCategoryData().initFromDbRow(self._oDb.fetchOne()); + self.dCache[idFailureCategory] = oEntry; + return oEntry; + + + # + # Helpers. + # + + def _readdEntry(self, uidAuthor, oData, tsEffective = None): + """ + Re-adds the FailureCategories entry. Used by addEntry, editEntry and removeEntry. + """ + if tsEffective is None: + tsEffective = self._oDb.getCurrentTimestamp(); + self._oDb.execute('INSERT INTO FailureCategories (\n' + ' uidAuthor,\n' + ' tsEffective,\n' + ' idFailureCategory,\n' + ' sShort,\n' + ' sFull)\n' + 'VALUES (%s, %s, ' + + ('DEFAULT' if oData.idFailureCategory is None else str(oData.idFailureCategory)) + + ', %s, %s)\n' + , ( uidAuthor, + tsEffective, + oData.sShort, + oData.sFull,) ); + return True; + + + def _historizeEntry(self, idFailureCategory, tsExpire = None): + """ Historizes the current entry. """ + if tsExpire is None: + tsExpire = self._oDb.getCurrentTimestamp(); + self._oDb.execute('UPDATE FailureCategories\n' + 'SET tsExpire = %s\n' + 'WHERE idFailureCategory = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , (tsExpire, idFailureCategory,)); + return True; + diff --git a/src/VBox/ValidationKit/testmanager/core/failurereason.py b/src/VBox/ValidationKit/testmanager/core/failurereason.py new file mode 100755 index 00000000..0d9294b0 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/failurereason.py @@ -0,0 +1,580 @@ +# -*- coding: utf-8 -*- +# $Id: failurereason.py $ + +""" +Test Manager - Failure Reasons. +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Standard Python imports. +import sys; + +# Validation Kit imports. +from testmanager.core.base import ModelDataBase, ModelLogicBase, TMRowNotFound, TMInvalidData, TMRowInUse, \ + AttributeChangeEntry, ChangeLogEntry; +from testmanager.core.useraccount import UserAccountLogic; + +# Python 3 hacks: +if sys.version_info[0] >= 3: + xrange = range; # pylint: disable=redefined-builtin,invalid-name + + +class FailureReasonData(ModelDataBase): + """ + Failure Reason Data. + """ + + ksIdAttr = 'idFailureReason'; + + ksParam_idFailureReason = 'FailureReasonData_idFailureReason' + ksParam_tsEffective = 'FailureReasonData_tsEffective' + ksParam_tsExpire = 'FailureReasonData_tsExpire' + ksParam_uidAuthor = 'FailureReasonData_uidAuthor' + ksParam_idFailureCategory = 'FailureReasonData_idFailureCategory' + ksParam_sShort = 'FailureReasonData_sShort' + ksParam_sFull = 'FailureReasonData_sFull' + ksParam_iTicket = 'FailureReasonData_iTicket' + ksParam_asUrls = 'FailureReasonData_asUrls' + + kasAllowNullAttributes = [ 'idFailureReason', 'tsEffective', 'tsExpire', + 'uidAuthor', 'iTicket', 'asUrls' ] + + def __init__(self): + ModelDataBase.__init__(self); + + # + # Initialize with defaults. + # See the database for explanations of each of these fields. + # + + self.idFailureReason = None + self.tsEffective = None + self.tsExpire = None + self.uidAuthor = None + self.idFailureCategory = None + self.sShort = None + self.sFull = None + self.iTicket = None + self.asUrls = None + + def initFromDbRow(self, aoRow): + """ + Re-initializes the data with a row from a SELECT * FROM FailureReasons. + + Returns self. Raises exception if the row is None or otherwise invalid. + """ + + if aoRow is None: + raise TMRowNotFound('Failure Reason not found.'); + + self.idFailureReason = aoRow[0] + self.tsEffective = aoRow[1] + self.tsExpire = aoRow[2] + self.uidAuthor = aoRow[3] + self.idFailureCategory = aoRow[4] + self.sShort = aoRow[5] + self.sFull = aoRow[6] + self.iTicket = aoRow[7] + self.asUrls = aoRow[8] + + return self; + + def initFromDbWithId(self, oDb, idFailureReason, tsNow = None, sPeriodBack = None): + """ + Initialize from the database, given the ID of a row. + """ + oDb.execute(self.formatSimpleNowAndPeriodQuery(oDb, + 'SELECT *\n' + 'FROM FailureReasons\n' + 'WHERE idFailureReason = %s\n' + , ( idFailureReason,), tsNow, sPeriodBack)); + aoRow = oDb.fetchOne() + if aoRow is None: + raise TMRowNotFound('idFailureReason=%s not found (tsNow=%s sPeriodBack=%s)' + % (idFailureReason, tsNow, sPeriodBack,)); + return self.initFromDbRow(aoRow); + + +class FailureReasonDataEx(FailureReasonData): + """ + Failure Reason Data, extended version that includes the category. + """ + + def __init__(self): + FailureReasonData.__init__(self); + self.oCategory = None; + self.oAuthor = None; + + def initFromDbRowEx(self, aoRow, oCategoryLogic, oUserAccountLogic): + """ + Re-initializes the data with a row from a SELECT * FROM FailureReasons. + + Returns self. Raises exception if the row is None or otherwise invalid. + """ + + self.initFromDbRow(aoRow); + self.oCategory = oCategoryLogic.cachedLookup(self.idFailureCategory); + self.oAuthor = oUserAccountLogic.cachedLookup(self.uidAuthor); + + return self; + + +class FailureReasonLogic(ModelLogicBase): # pylint: disable=too-few-public-methods + """ + Failure Reason logic. + """ + + def __init__(self, oDb): + ModelLogicBase.__init__(self, oDb) + self.dCache = None; + self.dCacheNameAndCat = None; + self.oCategoryLogic = None; + self.oUserAccountLogic = None; + + def fetchForListing(self, iStart, cMaxRows, tsNow, aiSortColumns = None): + """ + Fetches Failure Category records. + + Returns an array (list) of FailureReasonDataEx items, empty list if none. + Raises exception on error. + """ + _ = aiSortColumns; + self._ensureCachesPresent(); + + if tsNow is None: + self._oDb.execute('SELECT FailureReasons.*,\n' + ' FailureCategories.sShort AS sCategory\n' + 'FROM FailureReasons,\n' + ' FailureCategories\n' + 'WHERE FailureReasons.tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND FailureCategories.idFailureCategory = FailureReasons.idFailureCategory\n' + ' AND FailureCategories.tsExpire = \'infinity\'::TIMESTAMP\n' + 'ORDER BY sCategory ASC, sShort ASC\n' + 'LIMIT %s OFFSET %s\n' + , (cMaxRows, iStart,)); + else: + self._oDb.execute('SELECT FailureReasons.*,\n' + ' FailureCategories.sShort AS sCategory\n' + 'FROM FailureReasons,\n' + ' FailureCategories\n' + 'WHERE FailureReasons.tsExpire > %s\n' + ' AND FailureReasons.tsEffective <= %s\n' + ' AND FailureCategories.idFailureCategory = FailureReasons.idFailureCategory\n' + ' AND FailureReasons.tsExpire > %s\n' + ' AND FailureReasons.tsEffective <= %s\n' + 'ORDER BY sCategory ASC, sShort ASC\n' + 'LIMIT %s OFFSET %s\n' + , (tsNow, tsNow, tsNow, tsNow, cMaxRows, iStart,)); + + aoRows = [] + for aoRow in self._oDb.fetchAll(): + aoRows.append(FailureReasonDataEx().initFromDbRowEx(aoRow, self.oCategoryLogic, self.oUserAccountLogic)); + return aoRows + + def fetchForListingInCategory(self, iStart, cMaxRows, tsNow, idFailureCategory, aiSortColumns = None): + """ + Fetches Failure Category records. + + Returns an array (list) of FailureReasonDataEx items, empty list if none. + Raises exception on error. + """ + _ = aiSortColumns; + self._ensureCachesPresent(); + + if tsNow is None: + self._oDb.execute('SELECT *\n' + 'FROM FailureReasons\n' + 'WHERE tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND idFailureCategory = %s\n' + 'ORDER BY sShort ASC\n' + 'LIMIT %s OFFSET %s\n' + , ( idFailureCategory, cMaxRows, iStart,)); + else: + self._oDb.execute('SELECT *\n' + 'FROM FailureReasons\n' + 'WHERE idFailureCategory = %s\n' + ' AND tsExpire > %s\n' + ' AND tsEffective <= %s\n' + 'ORDER BY sShort ASC\n' + 'LIMIT %s OFFSET %s\n' + , ( idFailureCategory, tsNow, tsNow, cMaxRows, iStart,)); + + aoRows = [] + for aoRow in self._oDb.fetchAll(): + aoRows.append(FailureReasonDataEx().initFromDbRowEx(aoRow, self.oCategoryLogic, self.oUserAccountLogic)); + return aoRows + + + def fetchForSheriffByNamedCategory(self, sFailureCategory): + """ + Fetches the short names of the reasons in the named category. + + Returns array of strings. + Raises exception on error. + """ + self._oDb.execute('SELECT FailureReasons.sShort\n' + 'FROM FailureReasons,\n' + ' FailureCategories\n' + 'WHERE FailureReasons.tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND FailureReasons.idFailureCategory = FailureCategories.idFailureCategory\n' + ' AND FailureCategories.sShort = %s\n' + 'ORDER BY FailureReasons.sShort ASC\n' + , ( sFailureCategory,)); + return [aoRow[0] for aoRow in self._oDb.fetchAll()]; + + + def fetchForCombo(self, sFirstEntry = 'Select a failure reason', tsEffective = None): + """ + Gets the list of Failure Reasons for a combo box. + Returns an array of (value [idFailureReason], drop-down-name [sShort], + hover-text [sFull]) tuples. + """ + if tsEffective is None: + self._oDb.execute('SELECT fr.idFailureReason, CONCAT(fc.sShort, \' / \', fr.sShort) as sComboText, fr.sFull\n' + 'FROM FailureReasons fr,\n' + ' FailureCategories fc\n' + 'WHERE fr.idFailureCategory = fc.idFailureCategory\n' + ' AND fr.tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND fc.tsExpire = \'infinity\'::TIMESTAMP\n' + 'ORDER BY sComboText') + else: + self._oDb.execute('SELECT fr.idFailureReason, CONCAT(fc.sShort, \' / \', fr.sShort) as sComboText, fr.sFull\n' + 'FROM FailureReasons fr,\n' + ' FailureCategories fc\n' + 'WHERE fr.idFailureCategory = fc.idFailureCategory\n' + ' AND fr.tsExpire > %s\n' + ' AND fr.tsEffective <= %s\n' + ' AND fc.tsExpire > %s\n' + ' AND fc.tsEffective <= %s\n' + 'ORDER BY sComboText' + , (tsEffective, tsEffective, tsEffective, tsEffective)); + aoRows = self._oDb.fetchAll(); + return [(-1, sFirstEntry, '')] + aoRows; + + + def fetchForChangeLog(self, idFailureReason, iStart, cMaxRows, tsNow): # pylint: disable=too-many-locals + """ + Fetches change log entries for a failure reason. + + Returns an array of ChangeLogEntry instance and an indicator whether + there are more entries. + Raises exception on error. + """ + self._ensureCachesPresent(); + + if tsNow is None: + tsNow = self._oDb.getCurrentTimestamp(); + + # 1. Get a list of the relevant changes. + self._oDb.execute('SELECT * FROM FailureReasons WHERE idFailureReason = %s AND tsEffective <= %s\n' + 'ORDER BY tsEffective DESC\n' + 'LIMIT %s OFFSET %s\n' + , ( idFailureReason, tsNow, cMaxRows + 1, iStart, )); + aoRows = []; + for aoChange in self._oDb.fetchAll(): + aoRows.append(FailureReasonData().initFromDbRow(aoChange)); + + # 2. Calculate the changes. + aoEntries = []; + for i in xrange(0, len(aoRows) - 1): + oNew = aoRows[i]; + oOld = aoRows[i + 1]; + + aoChanges = []; + for sAttr in oNew.getDataAttributes(): + if sAttr not in [ 'tsEffective', 'tsExpire', 'uidAuthor', ]: + oOldAttr = getattr(oOld, sAttr); + oNewAttr = getattr(oNew, sAttr); + if oOldAttr != oNewAttr: + if sAttr == 'idFailureCategory': + oCat = self.oCategoryLogic.cachedLookup(oOldAttr); + if oCat is not None: + oOldAttr = '%s (%s)' % (oOldAttr, oCat.sShort, ); + oCat = self.oCategoryLogic.cachedLookup(oNewAttr); + if oCat is not None: + oNewAttr = '%s (%s)' % (oNewAttr, oCat.sShort, ); + aoChanges.append(AttributeChangeEntry(sAttr, oNewAttr, oOldAttr, str(oNewAttr), str(oOldAttr))); + + aoEntries.append(ChangeLogEntry(oNew.uidAuthor, None, oNew.tsEffective, oNew.tsExpire, oNew, oOld, aoChanges)); + + # If we're at the end of the log, add the initial entry. + if len(aoRows) <= cMaxRows and aoRows: + oNew = aoRows[-1]; + aoEntries.append(ChangeLogEntry(oNew.uidAuthor, None, oNew.tsEffective, oNew.tsExpire, oNew, None, [])); + + return (UserAccountLogic(self._oDb).resolveChangeLogAuthors(aoEntries), len(aoRows) > cMaxRows); + + + def getById(self, idFailureReason): + """Get Failure Reason data by idFailureReason""" + + self._oDb.execute('SELECT *\n' + 'FROM FailureReasons\n' + 'WHERE tsExpire = \'infinity\'::timestamp\n' + ' AND idFailureReason = %s;', (idFailureReason,)) + aRows = self._oDb.fetchAll() + if len(aRows) not in (0, 1): + raise self._oDb.integrityException( + 'Found more than one failure reasons with the same credentials. Database structure is corrupted.') + try: + return FailureReasonData().initFromDbRow(aRows[0]) + except IndexError: + return None + + + def addEntry(self, oData, uidAuthor, fCommit = False): + """ + Add a failure reason. + """ + # + # Validate. + # + dErrors = oData.validateAndConvert(self._oDb, oData.ksValidateFor_Add); + if dErrors: + raise TMInvalidData('addEntry invalid input: %s' % (dErrors,)); + + # + # Add the record. + # + self._readdEntry(uidAuthor, oData); + self._oDb.maybeCommit(fCommit); + return True; + + + def editEntry(self, oData, uidAuthor, fCommit = False): + """ + Modifies a failure reason. + """ + + # + # Validate inputs and read in the old(/current) data. + # + assert isinstance(oData, FailureReasonData); + dErrors = oData.validateAndConvert(self._oDb, oData.ksValidateFor_Edit); + if dErrors: + raise TMInvalidData('editEntry invalid input: %s' % (dErrors,)); + + oOldData = FailureReasonData().initFromDbWithId(self._oDb, oData.idFailureReason); + + # + # Update the data that needs updating. + # + if not oData.isEqualEx(oOldData, [ 'tsEffective', 'tsExpire', 'uidAuthor', ]): + self._historizeEntry(oData.idFailureReason); + self._readdEntry(uidAuthor, oData); + self._oDb.maybeCommit(fCommit); + return True; + + + def removeEntry(self, uidAuthor, idFailureReason, fCascade = False, fCommit = False): + """ + Deletes a failure reason. + """ + _ = fCascade; # too complicated for now. + + # + # Check whether it's being used by other tables and bitch if it is . + # We currently do not implement cascading. + # + self._oDb.execute('SELECT CONCAT(idBlacklisting, \' - blacklisting\')\n' + 'FROM BuildBlacklist\n' + 'WHERE idFailureReason = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + 'UNION\n' + 'SELECT CONCAT(idTestResult, \' - test result failure reason\')\n' + 'FROM TestResultFailures\n' + 'WHERE idFailureReason = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , (idFailureReason, idFailureReason,)); + aaoRows = self._oDb.fetchAll(); + if aaoRows: + raise TMRowInUse('Cannot remove failure reason %u because its being used by: %s' + % (idFailureReason, ', '.join(aoRow[0] for aoRow in aaoRows),)); + + # + # Do the job. + # + oData = FailureReasonData().initFromDbWithId(self._oDb, idFailureReason); + assert oData.idFailureReason == idFailureReason; + (tsCur, tsCurMinusOne) = self._oDb.getCurrentTimestamps(); + if oData.tsEffective not in (tsCur, tsCurMinusOne): + self._historizeEntry(idFailureReason, tsCurMinusOne); + self._readdEntry(uidAuthor, oData, tsCurMinusOne); + self._historizeEntry(idFailureReason); + self._oDb.maybeCommit(fCommit); + return True; + + + def cachedLookup(self, idFailureReason): + """ + Looks up the most recent FailureReasonDataEx object for idFailureReason + via an object cache. + + Returns a shared FailureReasonData object. None if not found. + Raises exception on DB error. + """ + if self.dCache is None: + self.dCache = self._oDb.getCache('FailureReasonDataEx'); + oEntry = self.dCache.get(idFailureReason, None); + if oEntry is None: + self._oDb.execute('SELECT *\n' + 'FROM FailureReasons\n' + 'WHERE idFailureReason = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , (idFailureReason, )); + if self._oDb.getRowCount() == 0: + # Maybe it was deleted, try get the last entry. + self._oDb.execute('SELECT *\n' + 'FROM FailureReasons\n' + 'WHERE idFailureReason = %s\n' + 'ORDER BY tsExpire DESC\n' + 'LIMIT 1\n' + , (idFailureReason, )); + elif self._oDb.getRowCount() > 1: + raise self._oDb.integrityException('%s infinity rows for %s' % (self._oDb.getRowCount(), idFailureReason)); + + if self._oDb.getRowCount() == 1: + self._ensureCachesPresent(); + oEntry = FailureReasonDataEx().initFromDbRowEx(self._oDb.fetchOne(), self.oCategoryLogic, + self.oUserAccountLogic); + self.dCache[idFailureReason] = oEntry; + return oEntry; + + + def cachedLookupByNameAndCategory(self, sName, sCategory): + """ + Looks up a failure reason by it's name and category. + + Should the request be ambigiuos, we'll return the oldest one. + + Returns a shared FailureReasonData object. None if not found. + Raises exception on DB error. + """ + if self.dCacheNameAndCat is None: + self.dCacheNameAndCat = self._oDb.getCache('FailureReasonDataEx-By-Name-And-Category'); + sKey = '%s:::%s' % (sName, sCategory,); + oEntry = self.dCacheNameAndCat.get(sKey, None); + if oEntry is None: + self._oDb.execute('SELECT *\n' + 'FROM FailureReasons,\n' + ' FailureCategories\n' + 'WHERE FailureReasons.sShort = %s\n' + ' AND FailureReasons.tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND FailureReasons.idFailureCategory = FailureCategories.idFailureCategory ' + ' AND FailureCategories.sShort = %s\n' + ' AND FailureCategories.tsExpire = \'infinity\'::TIMESTAMP\n' + 'ORDER BY FailureReasons.tsEffective\n' + , ( sName, sCategory)); + if self._oDb.getRowCount() == 0: + sLikeSucks = self._oDb.formatBindArgs( + 'SELECT *\n' + 'FROM FailureReasons,\n' + ' FailureCategories\n' + 'WHERE ( FailureReasons.sShort ILIKE @@@@@@@! %s !@@@@@@@\n' + ' OR FailureReasons.sFull ILIKE @@@@@@@! %s !@@@@@@@)\n' + ' AND FailureCategories.tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND FailureReasons.idFailureCategory = FailureCategories.idFailureCategory\n' + ' AND ( FailureCategories.sShort = %s\n' + ' OR FailureCategories.sFull = %s)\n' + ' AND FailureReasons.tsExpire = \'infinity\'::TIMESTAMP\n' + 'ORDER BY FailureReasons.tsEffective\n' + , ( sName, sName, sCategory, sCategory )); + sLikeSucks = sLikeSucks.replace('LIKE @@@@@@@! \'', 'LIKE \'%').replace('\' !@@@@@@@', '%\''); + self._oDb.execute(sLikeSucks); + if self._oDb.getRowCount() > 0: + self._ensureCachesPresent(); + oEntry = FailureReasonDataEx().initFromDbRowEx(self._oDb.fetchOne(), self.oCategoryLogic, + self.oUserAccountLogic); + self.dCacheNameAndCat[sKey] = oEntry; + if sName != oEntry.sShort or sCategory != oEntry.oCategory.sShort: + sKey2 = '%s:::%s' % (oEntry.sShort, oEntry.oCategory.sShort,); + self.dCacheNameAndCat[sKey2] = oEntry; + return oEntry; + + + # + # Helpers. + # + + def _readdEntry(self, uidAuthor, oData, tsEffective = None): + """ + Re-adds the FailureReasons entry. Used by addEntry, editEntry and removeEntry. + """ + if tsEffective is None: + tsEffective = self._oDb.getCurrentTimestamp(); + self._oDb.execute('INSERT INTO FailureReasons (\n' + ' uidAuthor,\n' + ' tsEffective,\n' + ' idFailureReason,\n' + ' idFailureCategory,\n' + ' sShort,\n' + ' sFull,\n' + ' iTicket,\n' + ' asUrls)\n' + 'VALUES (%s, %s, ' + + ( 'DEFAULT' if oData.idFailureReason is None else str(oData.idFailureReason) ) + + ', %s, %s, %s, %s, %s)\n' + , ( uidAuthor, + tsEffective, + oData.idFailureCategory, + oData.sShort, + oData.sFull, + oData.iTicket, + oData.asUrls,) ); + return True; + + + def _historizeEntry(self, idFailureReason, tsExpire = None): + """ Historizes the current entry. """ + if tsExpire is None: + tsExpire = self._oDb.getCurrentTimestamp(); + self._oDb.execute('UPDATE FailureReasons\n' + 'SET tsExpire = %s\n' + 'WHERE idFailureReason = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , (tsExpire, idFailureReason,)); + return True; + + + def _ensureCachesPresent(self): + """ Ensures we've got the cache references resolved. """ + if self.oCategoryLogic is None: + from testmanager.core.failurecategory import FailureCategoryLogic; + self.oCategoryLogic = FailureCategoryLogic(self._oDb); + if self.oUserAccountLogic is None: + self.oUserAccountLogic = UserAccountLogic(self._oDb); + return True; + diff --git a/src/VBox/ValidationKit/testmanager/core/globalresource.pgsql b/src/VBox/ValidationKit/testmanager/core/globalresource.pgsql new file mode 100644 index 00000000..44b5c473 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/globalresource.pgsql @@ -0,0 +1,118 @@ +-- $Id: globalresource.pgsql $ +--- @file +-- VBox Test Manager Database Stored Procedures. +-- + +-- +-- Copyright (C) 2006-2023 Oracle and/or its affiliates. +-- +-- This file is part of VirtualBox base platform packages, as +-- available from https://www.virtualbox.org. +-- +-- This program is free software; you can redistribute it and/or +-- modify it under the terms of the GNU General Public License +-- as published by the Free Software Foundation, in version 3 of the +-- License. +-- +-- This program is distributed in the hope that it will be useful, but +-- WITHOUT ANY WARRANTY; without even the implied warranty of +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +-- General Public License for more details. +-- +-- You should have received a copy of the GNU General Public License +-- along with this program; if not, see <https://www.gnu.org/licenses>. +-- +-- The contents of this file may alternatively be used under the terms +-- of the Common Development and Distribution License Version 1.0 +-- (CDDL), a copy of it is provided in the "COPYING.CDDL" file included +-- in the VirtualBox distribution, in which case the provisions of the +-- CDDL are applicable instead of those of the GPL. +-- +-- You may elect to license modified versions of this file under the +-- terms and conditions of either the GPL or the CDDL or both. +-- +-- SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +-- + +\set ON_ERROR_STOP 1 +\connect testmanager; + +-- Args: uidAuthor, sName, sDescription, fEnabled +CREATE OR REPLACE function add_globalresource(integer, text, text, bool) RETURNS integer AS $$ + DECLARE + _idGlobalRsrc integer; + _uidAuthor ALIAS FOR $1; + _sName ALIAS FOR $2; + _sDescription ALIAS FOR $3; + _fEnabled ALIAS FOR $4; + BEGIN + -- Check if Global Resource name is unique + IF EXISTS(SELECT * FROM GlobalResources + WHERE sName=_sName AND + tsExpire='infinity'::timestamp) THEN + RAISE EXCEPTION 'Duplicate Global Resource name'; + END IF; + INSERT INTO GlobalResources (uidAuthor, sName, sDescription, fEnabled) + VALUES (_uidAuthor, _sName, _sDescription, _fEnabled) RETURNING idGlobalRsrc INTO _idGlobalRsrc; + RETURN _idGlobalRsrc; + END; +$$ LANGUAGE plpgsql; + +-- Args: uidAuthor, idGlobalRsrc +CREATE OR REPLACE function del_globalresource(integer, integer) RETURNS VOID AS $$ + DECLARE + _uidAuthor ALIAS FOR $1; + _idGlobalRsrc ALIAS FOR $2; + BEGIN + + -- Check if record exist + IF NOT EXISTS(SELECT * FROM GlobalResources WHERE idGlobalRsrc=_idGlobalRsrc AND tsExpire='infinity'::timestamp) THEN + RAISE EXCEPTION 'Global resource (%) does not exist', _idGlobalRsrc; + END IF; + + -- Historize record: GlobalResources + UPDATE GlobalResources + SET tsExpire=CURRENT_TIMESTAMP, + uidAuthor=_uidAuthor + WHERE idGlobalRsrc=_idGlobalRsrc AND + tsExpire='infinity'::timestamp; + + + -- Delete record: GlobalResourceStatuses + DELETE FROM GlobalResourceStatuses WHERE idGlobalRsrc=_idGlobalRsrc; + + -- Historize record: TestCaseGlobalRsrcDeps + UPDATE TestCaseGlobalRsrcDeps + SET tsExpire=CURRENT_TIMESTAMP, + uidAuthor=_uidAuthor + WHERE idGlobalRsrc=_idGlobalRsrc AND + tsExpire='infinity'::timestamp; + + END; +$$ LANGUAGE plpgsql; + +-- Args: uidAuthor, idGlobalRsrc, sName, sDescription, fEnabled +CREATE OR REPLACE function update_globalresource(integer, integer, text, text, bool) RETURNS VOID AS $$ + DECLARE + _uidAuthor ALIAS FOR $1; + _idGlobalRsrc ALIAS FOR $2; + _sName ALIAS FOR $3; + _sDescription ALIAS FOR $4; + _fEnabled ALIAS FOR $5; + BEGIN + -- Hostorize record + UPDATE GlobalResources + SET tsExpire=CURRENT_TIMESTAMP + WHERE idGlobalRsrc=_idGlobalRsrc AND + tsExpire='infinity'::timestamp; + -- Check if Global Resource name is unique + IF EXISTS(SELECT * FROM GlobalResources + WHERE sName=_sName AND + tsExpire='infinity'::timestamp) THEN + RAISE EXCEPTION 'Duplicate Global Resource name'; + END IF; + -- Add new record + INSERT INTO GlobalResources(uidAuthor, idGlobalRsrc, sName, sDescription, fEnabled) + VALUES (_uidAuthor, _idGlobalRsrc, _sName, _sDescription, _fEnabled); + END; +$$ LANGUAGE plpgsql; diff --git a/src/VBox/ValidationKit/testmanager/core/globalresource.py b/src/VBox/ValidationKit/testmanager/core/globalresource.py new file mode 100755 index 00000000..bd6f0e9e --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/globalresource.py @@ -0,0 +1,328 @@ +# -*- coding: utf-8 -*- +# $Id: globalresource.py $ + +""" +Test Manager - Global Resources. +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Standard python imports. +import unittest; + +# Validation Kit imports. +from testmanager.core.base import ModelDataBase, ModelDataBaseTestCase, ModelLogicBase, TMRowNotFound; + + +class GlobalResourceData(ModelDataBase): + """ + Global resource data + """ + + ksIdAttr = 'idGlobalRsrc'; + + ksParam_idGlobalRsrc = 'GlobalResource_idGlobalRsrc' + ksParam_tsEffective = 'GlobalResource_tsEffective' + ksParam_tsExpire = 'GlobalResource_tsExpire' + ksParam_uidAuthor = 'GlobalResource_uidAuthor' + ksParam_sName = 'GlobalResource_sName' + ksParam_sDescription = 'GlobalResource_sDescription' + ksParam_fEnabled = 'GlobalResource_fEnabled' + + kasAllowNullAttributes = ['idGlobalRsrc', 'tsEffective', 'tsExpire', 'uidAuthor', 'sDescription' ]; + kcchMin_sName = 2; + kcchMax_sName = 64; + + def __init__(self): + ModelDataBase.__init__(self); + + # + # Initialize with defaults. + # See the database for explanations of each of these fields. + # + self.idGlobalRsrc = None; + self.tsEffective = None; + self.tsExpire = None; + self.uidAuthor = None; + self.sName = None; + self.sDescription = None; + self.fEnabled = False + + def initFromDbRow(self, aoRow): + """ + Reinitialize from a SELECT * FROM GlobalResources row. + Returns self. Raises exception if no row. + """ + if aoRow is None: + raise TMRowNotFound('Global resource not found.') + + self.idGlobalRsrc = aoRow[0] + self.tsEffective = aoRow[1] + self.tsExpire = aoRow[2] + self.uidAuthor = aoRow[3] + self.sName = aoRow[4] + self.sDescription = aoRow[5] + self.fEnabled = aoRow[6] + return self + + def initFromDbWithId(self, oDb, idGlobalRsrc, tsNow = None, sPeriodBack = None): + """ + Initialize the object from the database. + """ + oDb.execute(self.formatSimpleNowAndPeriodQuery(oDb, + 'SELECT *\n' + 'FROM GlobalResources\n' + 'WHERE idGlobalRsrc = %s\n' + , ( idGlobalRsrc,), tsNow, sPeriodBack)); + aoRow = oDb.fetchOne() + if aoRow is None: + raise TMRowNotFound('idGlobalRsrc=%s not found (tsNow=%s sPeriodBack=%s)' % (idGlobalRsrc, tsNow, sPeriodBack,)); + return self.initFromDbRow(aoRow); + + def isEqual(self, oOther): + """ + Compares two instances. + """ + return self.idGlobalRsrc == oOther.idGlobalRsrc \ + and str(self.tsEffective) == str(oOther.tsEffective) \ + and str(self.tsExpire) == str(oOther.tsExpire) \ + and self.uidAuthor == oOther.uidAuthor \ + and self.sName == oOther.sName \ + and self.sDescription == oOther.sDescription \ + and self.fEnabled == oOther.fEnabled + + +class GlobalResourceLogic(ModelLogicBase): + """ + Global resource logic. + """ + + def __init__(self, oDb): + ModelLogicBase.__init__(self, oDb) + self.dCache = None; + + def fetchForListing(self, iStart, cMaxRows, tsNow, aiSortColumns = None): + """ + Returns an array (list) of FailureReasonData items, empty list if none. + Raises exception on error. + """ + _ = aiSortColumns; + + if tsNow is None: + self._oDb.execute('SELECT *\n' + 'FROM GlobalResources\n' + 'WHERE tsExpire = \'infinity\'::TIMESTAMP\n' + 'ORDER BY idGlobalRsrc DESC\n' + 'LIMIT %s OFFSET %s\n' + , (cMaxRows, iStart,)); + else: + self._oDb.execute('SELECT *\n' + 'FROM GlobalResources\n' + 'WHERE tsExpire > %s\n' + ' AND tsEffective <= %s\n' + 'ORDER BY idGlobalRsrc DESC\n' + 'LIMIT %s OFFSET %s\n' + , (tsNow, tsNow, cMaxRows, iStart,)) + + aoRows = [] + for aoRow in self._oDb.fetchAll(): + aoRows.append(GlobalResourceData().initFromDbRow(aoRow)) + return aoRows + + + def cachedLookup(self, idGlobalRsrc): + """ + Looks up the most recent GlobalResourceData object for idGlobalRsrc + via an object cache. + + Returns a shared GlobalResourceData object. None if not found. + Raises exception on DB error. + """ + if self.dCache is None: + self.dCache = self._oDb.getCache('GlobalResourceData'); + oEntry = self.dCache.get(idGlobalRsrc, None); + if oEntry is None: + self._oDb.execute('SELECT *\n' + 'FROM GlobalResources\n' + 'WHERE idGlobalRsrc = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , (idGlobalRsrc, )); + if self._oDb.getRowCount() == 0: + # Maybe it was deleted, try get the last entry. + self._oDb.execute('SELECT *\n' + 'FROM GlobalResources\n' + 'WHERE idGlobalRsrc = %s\n' + 'ORDER BY tsExpire DESC\n' + 'LIMIT 1\n' + , (idGlobalRsrc, )); + elif self._oDb.getRowCount() > 1: + raise self._oDb.integrityException('%s infinity rows for %s' % (self._oDb.getRowCount(), idGlobalRsrc)); + + if self._oDb.getRowCount() == 1: + aaoRow = self._oDb.fetchOne(); + oEntry = GlobalResourceData(); + oEntry.initFromDbRow(aaoRow); + self.dCache[idGlobalRsrc] = oEntry; + return oEntry; + + + def getAll(self, tsEffective = None): + """ + Gets all global resources. + + Returns an array of GlobalResourceData instances on success (can be + empty). Raises exception on database error. + """ + if tsEffective is not None: + self._oDb.execute('SELECT *\n' + 'FROM GlobalResources\n' + 'WHERE tsExpire > %s\n' + ' AND tsEffective <= %s\n' + , (tsEffective, tsEffective)); + else: + self._oDb.execute('SELECT *\n' + 'FROM GlobalResources\n' + 'WHERE tsExpire = \'infinity\'::TIMESTAMP\n'); + aaoRows = self._oDb.fetchAll(); + aoRet = []; + for aoRow in aaoRows: + aoRet.append(GlobalResourceData().initFromDbRow(aoRow)); + + return aoRet; + + def addGlobalResource(self, uidAuthor, oData): + """Add Global Resource DB record""" + self._oDb.execute('SELECT * FROM add_globalresource(%s, %s, %s, %s);', + (uidAuthor, + oData.sName, + oData.sDescription, + oData.fEnabled)) + self._oDb.commit() + return True + + def editGlobalResource(self, uidAuthor, idGlobalRsrc, oData): + """Modify Global Resource DB record""" + # Check if anything has been changed + oGlobalResourcesDataOld = self.getById(idGlobalRsrc) + if oGlobalResourcesDataOld.isEqual(oData): + # Nothing has been changed, do nothing + return True + + self._oDb.execute('SELECT * FROM update_globalresource(%s, %s, %s, %s, %s);', + (uidAuthor, + idGlobalRsrc, + oData.sName, + oData.sDescription, + oData.fEnabled)) + self._oDb.commit() + return True + + def remove(self, uidAuthor, idGlobalRsrc): + """Delete Global Resource DB record""" + self._oDb.execute('SELECT * FROM del_globalresource(%s, %s);', + (uidAuthor, idGlobalRsrc)) + self._oDb.commit() + return True + + def getById(self, idGlobalRsrc): + """ + Get global resource record by its id + """ + self._oDb.execute('SELECT *\n' + 'FROM GlobalResources\n' + 'WHERE tsExpire = \'infinity\'::timestamp\n' + ' AND idGlobalRsrc=%s;', (idGlobalRsrc,)) + + aRows = self._oDb.fetchAll() + if len(aRows) not in (0, 1): + raise self._oDb.integrityException('Duplicate global resource entry with ID %u (current)' % (idGlobalRsrc,)); + try: + return GlobalResourceData().initFromDbRow(aRows[0]) + except IndexError: + raise TMRowNotFound('Global resource not found.') + + def allocateResources(self, idTestBox, aoGlobalRsrcs, fCommit = False): + """ + Allocates the given global resource. + + Returns True of successfully allocated the resources, False if not. + May raise exception on DB error. + """ + # Quit quickly if there is nothing to alloocate. + if not aoGlobalRsrcs: + return True; + + # + # Note! Someone else might have allocated the resources since the + # scheduler check that they were available. In such case we + # need too quietly rollback and return FALSE. + # + self._oDb.execute('SAVEPOINT allocateResources'); + + for oGlobalRsrc in aoGlobalRsrcs: + try: + self._oDb.execute('INSERT INTO GlobalResourceStatuses (idGlobalRsrc, idTestBox)\n' + 'VALUES (%s, %s)', (oGlobalRsrc.idGlobalRsrc, idTestBox, ) ); + except self._oDb.oXcptError: + self._oDb.execute('ROLLBACK TO SAVEPOINT allocateResources'); + return False; + + self._oDb.execute('RELEASE SAVEPOINT allocateResources'); + self._oDb.maybeCommit(fCommit); + return True; + + def freeGlobalResourcesByTestBox(self, idTestBox, fCommit = False): + """ + Frees all global resources own by the given testbox. + Returns True. May raise exception on DB error. + """ + self._oDb.execute('DELETE FROM GlobalResourceStatuses\n' + 'WHERE idTestBox = %s\n', (idTestBox, ) ); + self._oDb.maybeCommit(fCommit); + return True; + +# +# Unit testing. +# + +# pylint: disable=missing-docstring +class GlobalResourceDataTestCase(ModelDataBaseTestCase): + def setUp(self): + self.aoSamples = [GlobalResourceData(),]; + +if __name__ == '__main__': + unittest.main(); + # not reached. + diff --git a/src/VBox/ValidationKit/testmanager/core/report.py b/src/VBox/ValidationKit/testmanager/core/report.py new file mode 100755 index 00000000..f07e75e8 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/report.py @@ -0,0 +1,1307 @@ +# -*- coding: utf-8 -*- +# $Id: report.py $ + +""" +Test Manager - Report models. +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Standard Python imports. +import sys; + +# Validation Kit imports. +from testmanager.core.base import ModelLogicBase, TMExceptionBase; +from testmanager.core.build import BuildCategoryData; +from testmanager.core.dbobjcache import DatabaseObjCache; +from testmanager.core.failurereason import FailureReasonLogic; +from testmanager.core.testbox import TestBoxLogic, TestBoxData; +from testmanager.core.testcase import TestCaseLogic; +from testmanager.core.testcaseargs import TestCaseArgsLogic; +from testmanager.core.testresults import TestResultLogic, TestResultFilter; +from common import constants; + +# Python 3 hacks: +if sys.version_info[0] >= 3: + xrange = range; # pylint: disable=redefined-builtin,invalid-name + + + +class ReportFilter(TestResultFilter): + """ + Same as TestResultFilter for now. + """ + + def __init__(self): + TestResultFilter.__init__(self); + + + +class ReportModelBase(ModelLogicBase): # pylint: disable=too-few-public-methods + """ + Something all report logic(/miner) classes inherit from. + """ + + ## @name Report subjects + ## @{ + ksSubEverything = 'Everything'; + ksSubSchedGroup = 'SchedGroup'; + ksSubTestGroup = 'TestGroup'; + ksSubTestCase = 'TestCase'; + ksSubTestCaseArgs = 'TestCaseArgs'; + ksSubTestBox = 'TestBox'; + ksSubBuild = 'Build'; + ## @} + kasSubjects = [ ksSubEverything, ksSubSchedGroup, ksSubTestGroup, ksSubTestCase, ksSubTestBox, ksSubBuild, ]; + + + ## @name TestStatus_T + # @{ + ksTestStatus_Running = 'running'; + ksTestStatus_Success = 'success'; + ksTestStatus_Skipped = 'skipped'; + ksTestStatus_BadTestBox = 'bad-testbox'; + ksTestStatus_Aborted = 'aborted'; + ksTestStatus_Failure = 'failure'; + ksTestStatus_TimedOut = 'timed-out'; + ksTestStatus_Rebooted = 'rebooted'; + ## @} + + + def __init__(self, oDb, tsNow, cPeriods, cHoursPerPeriod, sSubject, aidSubjects, oFilter): + ModelLogicBase.__init__(self, oDb); + # Public so the report generator can easily access them. + self.tsNow = tsNow; # (Can be None.) + self.__tsNowDateTime = None; + self.cPeriods = cPeriods; + self.cHoursPerPeriod = cHoursPerPeriod; + self.sSubject = sSubject; + self.aidSubjects = aidSubjects; + self.oFilter = oFilter; + if self.oFilter is None: + class DummyFilter(object): + """ Dummy """ + def getTableJoins(self, sExtraIndent = '', iOmit = -1, dOmitTables = None): + """ Dummy """ + _ = sExtraIndent; _ = iOmit; _ = dOmitTables; # pylint: disable=redefined-variable-type + return ''; + def getWhereConditions(self, sExtraIndent = '', iOmit = -1): + """ Dummy """ + _ = sExtraIndent; _ = iOmit; # pylint: disable=redefined-variable-type + return ''; + def isJoiningWithTable(self, sTable): + """ Dummy """; + _ = sTable; + return False; + self.oFilter = DummyFilter(); + + def getExtraSubjectTables(self): + """ + Returns a list of additional tables needed by the subject. + """ + return []; + + def getExtraSubjectWhereExpr(self): + """ + Returns additional WHERE expression relating to the report subject. It starts + with an AND so that it can simply be appended to the WHERE clause. + """ + if self.sSubject == self.ksSubEverything: + return ''; + + if self.sSubject == self.ksSubSchedGroup: + sWhere = ' AND TestSets.idSchedGroup'; + elif self.sSubject == self.ksSubTestGroup: + sWhere = ' AND TestSets.idTestGroup'; + elif self.sSubject == self.ksSubTestCase: + sWhere = ' AND TestSets.idTestCase'; + elif self.sSubject == self.ksSubTestCaseArgs: + sWhere = ' AND TestSets.idTestCaseArgs'; + elif self.sSubject == self.ksSubTestBox: + sWhere = ' AND TestSets.idTestBox'; + elif self.sSubject == self.ksSubBuild: + sWhere = ' AND TestSets.idBuild'; + else: + raise TMExceptionBase(self.sSubject); + + if len(self.aidSubjects) == 1: + sWhere += self._oDb.formatBindArgs(' = %s\n', (self.aidSubjects[0],)); + else: + assert self.aidSubjects; + sWhere += self._oDb.formatBindArgs(' IN (%s', (self.aidSubjects[0],)); + for i in range(1, len(self.aidSubjects)): + sWhere += self._oDb.formatBindArgs(', %s', (self.aidSubjects[i],)); + sWhere += ')\n'; + + return sWhere; + + def getNowAsDateTime(self): + """ Returns a datetime instance corresponding to tsNow. """ + if self.__tsNowDateTime is None: + if self.tsNow is None: + self.__tsNowDateTime = self._oDb.getCurrentTimestamp(); + else: + self._oDb.execute('SELECT %s::TIMESTAMP WITH TIME ZONE', (self.tsNow,)); + self.__tsNowDateTime = self._oDb.fetchOne()[0]; + return self.__tsNowDateTime; + + def getPeriodStart(self, iPeriod): + """ Gets the python timestamp for the start of the given period. """ + from datetime import timedelta; + cHoursStart = (self.cPeriods - iPeriod ) * self.cHoursPerPeriod; + return self.getNowAsDateTime() - timedelta(hours = cHoursStart); + + def getPeriodEnd(self, iPeriod): + """ Gets the python timestamp for the end of the given period. """ + from datetime import timedelta; + cHoursEnd = (self.cPeriods - iPeriod - 1) * self.cHoursPerPeriod; + return self.getNowAsDateTime() - timedelta(hours = cHoursEnd); + + def getExtraWhereExprForPeriod(self, iPeriod): + """ + Returns additional WHERE expression for getting test sets for the + specified period. It starts with an AND so that it can simply be + appended to the WHERE clause. + """ + if self.tsNow is None: + sNow = 'CURRENT_TIMESTAMP'; + else: + sNow = self._oDb.formatBindArgs('%s::TIMESTAMP', (self.tsNow,)); + + cHoursStart = (self.cPeriods - iPeriod ) * self.cHoursPerPeriod; + cHoursEnd = (self.cPeriods - iPeriod - 1) * self.cHoursPerPeriod; + if cHoursEnd == 0: + return ' AND TestSets.tsDone >= (%s - interval \'%u hours\')\n' \ + ' AND TestSets.tsDone < %s\n' \ + % (sNow, cHoursStart, sNow); + return ' AND TestSets.tsDone >= (%s - interval \'%u hours\')\n' \ + ' AND TestSets.tsDone < (%s - interval \'%u hours\')\n' \ + % (sNow, cHoursStart, sNow, cHoursEnd); + + def getPeriodDesc(self, iPeriod): + """ + Returns the period description, usually for graph data. + """ + if iPeriod == 0: + return 'now' if self.tsNow is None else 'then'; + sTerm = 'ago' if self.tsNow is None else 'earlier'; + if self.cHoursPerPeriod == 24: + return '%dd %s' % (iPeriod, sTerm, ); + if (iPeriod * self.cHoursPerPeriod) % 24 == 0: + return '%dd %s' % (iPeriod * self.cHoursPerPeriod / 24, sTerm, ); + return '%dh %s' % (iPeriod * self.cHoursPerPeriod, sTerm); + + def getStraightPeriodDesc(self, iPeriod): + """ + Returns the period description, usually for graph data. + """ + iWickedPeriod = self.cPeriods - iPeriod - 1; + return self.getPeriodDesc(iWickedPeriod); + + +# +# Data structures produced and returned by the ReportLazyModel. +# + +class ReportTransientBase(object): + """ Details on the test where a problem was first/last seen. """ + def __init__(self, idBuild, iRevision, sRepository, idTestSet, idTestResult, tsDone, # pylint: disable=too-many-arguments + iPeriod, fEnter, idSubject, oSubject): + self.idBuild = idBuild; # Build ID. + self.iRevision = iRevision; # SVN revision for build. + self.sRepository = sRepository; # SVN repository for build. + self.idTestSet = idTestSet; # Test set. + self.idTestResult = idTestResult; # Test result. + self.tsDone = tsDone; # When the test set was done. + self.iPeriod = iPeriod; # Data set period. + self.fEnter = fEnter; # True if enter event, False if leave event. + self.idSubject = idSubject; + self.oSubject = oSubject; + +class ReportFailureReasonTransient(ReportTransientBase): + """ Details on the test where a failure reason was first/last seen. """ + def __init__(self, idBuild, iRevision, sRepository, idTestSet, idTestResult, tsDone, # pylint: disable=too-many-arguments + iPeriod, fEnter, oReason): + ReportTransientBase.__init__(self, idBuild, iRevision, sRepository, idTestSet, idTestResult, tsDone, iPeriod, fEnter, + oReason.idFailureReason, oReason); + self.oReason = oReason; # FailureReasonDataEx + + +class ReportHitRowBase(object): + """ A row in a period. """ + def __init__(self, idSubject, oSubject, cHits, tsMin = None, tsMax = None): + self.idSubject = idSubject; + self.oSubject = oSubject; + self.cHits = cHits; + self.tsMin = tsMin; + self.tsMax = tsMax; + +class ReportHitRowWithTotalBase(ReportHitRowBase): + """ A row in a period. """ + def __init__(self, idSubject, oSubject, cHits, cTotal, tsMin = None, tsMax = None): + ReportHitRowBase.__init__(self, idSubject, oSubject, cHits, tsMin, tsMax) + self.cTotal = cTotal; + self.uPct = cHits * 100 / cTotal; + +class ReportFailureReasonRow(ReportHitRowBase): + """ The account of one failure reason for a period. """ + def __init__(self, aoRow, oReason): + ReportHitRowBase.__init__(self, aoRow[0], oReason, aoRow[1], aoRow[2], aoRow[3]); + self.idFailureReason = aoRow[0]; + self.oReason = oReason; # FailureReasonDataEx + + +class ReportPeriodBase(object): + """ A period in ReportFailureReasonSet. """ + def __init__(self, oSet, iPeriod, sDesc, tsFrom, tsTo): + self.oSet = oSet # Reference to the parent ReportSetBase derived object. + self.iPeriod = iPeriod; # Period number in the set. + self.sDesc = sDesc; # Short period description. + self.tsStart = tsFrom; # Start of the period. + self.tsEnd = tsTo; # End of the period (exclusive). + self.tsMin = tsTo; # The earlierst hit of the period (only valid for cHits > 0). + self.tsMax = tsFrom; # The latest hit of the period (only valid for cHits > 0). + self.aoRows = []; # Rows in order the database returned them (ReportHitRowBase descendant). + self.dRowsById = {}; # Same as aoRows but indexed by object ID (see ReportSetBase::sIdAttr). + self.dFirst = {}; # The subjects seen for the first time - data object, keyed by ID. + self.dLast = {}; # The subjects seen for the last time - data object, keyed by ID. + self.cHits = 0; # Total number of hits in this period. + self.cMaxHits = 0; # Max hits in a row. + self.cMinHits = 99999999; # Min hits in a row (only valid for cHits > 0). + + def appendRow(self, oRow, idRow, oData): + """ Adds a row. """ + assert isinstance(oRow, ReportHitRowBase); + self.aoRows.append(oRow); + self.dRowsById[idRow] = oRow; + if idRow not in self.oSet.dSubjects: + self.oSet.dSubjects[idRow] = oData; + self._doStatsForRow(oRow, idRow, oData); + + def _doStatsForRow(self, oRow, idRow, oData): + """ Does the statistics for a row. Helper for appendRow as well as helpRecalcStats. """ + if oRow.tsMin is not None and oRow.tsMin < self.tsMin: + self.tsMin = oRow.tsMin; + if oRow.tsMax is not None and oRow.tsMax < self.tsMax: + self.tsMax = oRow.tsMax; + + self.cHits += oRow.cHits; + if oRow.cHits > self.cMaxHits: + self.cMaxHits = oRow.cHits; + if oRow.cHits < self.cMinHits: + self.cMinHits = oRow.cHits; + + if idRow in self.oSet.dcHitsPerId: + self.oSet.dcHitsPerId[idRow] += oRow.cHits; + else: + self.oSet.dcHitsPerId[idRow] = oRow.cHits; + + if oRow.cHits > 0: + if idRow not in self.oSet.diPeriodFirst: + self.dFirst[idRow] = oData; + self.oSet.diPeriodFirst[idRow] = self.iPeriod; + self.oSet.diPeriodLast[idRow] = self.iPeriod; + + def helperSetRecalcStats(self): + """ Recalc the statistics (do resetStats first on set). """ + for idRow, oRow in self.dRowsById.items(): + self._doStatsForRow(oRow, idRow, self.oSet.dSubjects[idRow]); + + def helperSetResetStats(self): + """ Resets the statistics. """ + self.tsMin = self.tsEnd; + self.tsMax = self.tsStart; + self.cHits = 0; + self.cMaxHits = 0; + self.cMinHits = 99999999; + self.dFirst = {}; + self.dLast = {}; + + def helperSetDeleteKeyFromSet(self, idKey): + """ Helper for ReportPeriodSetBase::deleteKey """ + if idKey in self.dRowsById: + oRow = self.dRowsById[idKey]; + self.aoRows.remove(oRow); + del self.dRowsById[idKey] + self.cHits -= oRow.cHits; + if idKey in self.dFirst: + del self.dFirst[idKey]; + if idKey in self.dLast: + del self.dLast[idKey]; + +class ReportPeriodWithTotalBase(ReportPeriodBase): + """ In addition to the cHits, we also have a total to relate it too. """ + def __init__(self, oSet, iPeriod, sDesc, tsFrom, tsTo): + ReportPeriodBase.__init__(self, oSet, iPeriod, sDesc, tsFrom, tsTo); + self.cTotal = 0; + self.cMaxTotal = 0; + self.cMinTotal = 99999999; + self.uMaxPct = 0; # Max percentage in a row (100 = 100%). + + def _doStatsForRow(self, oRow, idRow, oData): + assert isinstance(oRow, ReportHitRowWithTotalBase); + super(ReportPeriodWithTotalBase, self)._doStatsForRow(oRow, idRow, oData); + self.cTotal += oRow.cTotal; + if oRow.cTotal > self.cMaxTotal: + self.cMaxTotal = oRow.cTotal; + if oRow.cTotal < self.cMinTotal: + self.cMinTotal = oRow.cTotal; + + if oRow.uPct > self.uMaxPct: + self.uMaxPct = oRow.uPct; + + if idRow in self.oSet.dcTotalPerId: + self.oSet.dcTotalPerId[idRow] += oRow.cTotal; + else: + self.oSet.dcTotalPerId[idRow] = oRow.cTotal; + + def helperSetResetStats(self): + super(ReportPeriodWithTotalBase, self).helperSetResetStats(); + self.cTotal = 0; + self.cMaxTotal = 0; + self.cMinTotal = 99999999; + self.uMaxPct = 0; + +class ReportFailureReasonPeriod(ReportPeriodBase): + """ A period in ReportFailureReasonSet. """ + def __init__(self, oSet, iPeriod, sDesc, tsFrom, tsTo): + ReportPeriodBase.__init__(self, oSet, iPeriod, sDesc, tsFrom, tsTo); + self.cWithoutReason = 0; # Number of failed test sets without any assigned reason. + + + +class ReportPeriodSetBase(object): + """ Period data set base class. """ + def __init__(self, sIdAttr): + self.sIdAttr = sIdAttr; # The name of the key attribute. Mainly for documentation purposes. + self.aoPeriods = []; # Periods (ReportPeriodBase descendant) in ascending order (time wise). + self.dSubjects = {}; # The subject data objects, keyed by the subject ID. + self.dcHitsPerId = {}; # Sum hits per subject ID (key). + self.cHits = 0; # Sum number of hits in all periods and all reasons. + self.cMaxHits = 0; # Max hits in a row. + self.cMinHits = 99999999; # Min hits in a row. + self.cMaxRows = 0; # Max number of rows in a period. + self.cMinRows = 99999999; # Min number of rows in a period. + self.diPeriodFirst = {}; # The period number a reason was first seen (keyed by subject ID). + self.diPeriodLast = {}; # The period number a reason was last seen (keyed by subject ID). + self.aoEnterInfo = []; # Array of ReportTransientBase children order by iRevision. Excludes + # the first period of course. (Child class populates this.) + self.aoLeaveInfo = []; # Array of ReportTransientBase children order in descending order by + # iRevision. Excludes the last priod. (Child class populates this.) + + def appendPeriod(self, oPeriod): + """ Appends a period to the set. """ + assert isinstance(oPeriod, ReportPeriodBase); + self.aoPeriods.append(oPeriod); + self._doStatsForPeriod(oPeriod); + + def _doStatsForPeriod(self, oPeriod): + """ Worker for appendPeriod and recalcStats. """ + self.cHits += oPeriod.cHits; + if oPeriod.cMaxHits > self.cMaxHits: + self.cMaxHits = oPeriod.cMaxHits; + if oPeriod.cMinHits < self.cMinHits: + self.cMinHits = oPeriod.cMinHits; + + if len(oPeriod.aoRows) > self.cMaxRows: + self.cMaxRows = len(oPeriod.aoRows); + if len(oPeriod.aoRows) < self.cMinRows: + self.cMinRows = len(oPeriod.aoRows); + + def recalcStats(self): + """ Recalculates the statistics. ASSUMES finalizePass1 hasn't been done yet. """ + self.cHits = 0; + self.cMaxHits = 0; + self.cMinHits = 99999999; + self.cMaxRows = 0; + self.cMinRows = 99999999; + self.diPeriodFirst = {}; + self.diPeriodLast = {}; + self.dcHitsPerId = {}; + for oPeriod in self.aoPeriods: + oPeriod.helperSetResetStats(); + + for oPeriod in self.aoPeriods: + oPeriod.helperSetRecalcStats(); + self._doStatsForPeriod(oPeriod); + + def deleteKey(self, idKey): + """ Deletes a key from the set. May leave cMaxHits and cMinHits with outdated values. """ + self.cHits -= self.dcHitsPerId[idKey]; + del self.dcHitsPerId[idKey]; + if idKey in self.diPeriodFirst: + del self.diPeriodFirst[idKey]; + if idKey in self.diPeriodLast: + del self.diPeriodLast[idKey]; + if idKey in self.aoEnterInfo: + del self.aoEnterInfo[idKey]; + if idKey in self.aoLeaveInfo: + del self.aoLeaveInfo[idKey]; + del self.dSubjects[idKey]; + for oPeriod in self.aoPeriods: + oPeriod.helperSetDeleteKeyFromSet(idKey); + + def pruneRowsWithZeroSumHits(self): + """ Discards rows with zero sum hits across all periods. Works around lazy selects counting both totals and hits. """ + cDeleted = 0; + aidKeys = list(self.dcHitsPerId); + for idKey in aidKeys: + if self.dcHitsPerId[idKey] == 0: + self.deleteKey(idKey); + cDeleted += 1; + if cDeleted > 0: + self.recalcStats(); + return cDeleted; + + def finalizePass1(self): + """ Finished all but aoEnterInfo and aoLeaveInfo. """ + # All we need to do here is to populate the dLast members. + for idKey, iPeriod in self.diPeriodLast.items(): + self.aoPeriods[iPeriod].dLast[idKey] = self.dSubjects[idKey]; + return self; + + def finalizePass2(self): + """ Called after aoEnterInfo and aoLeaveInfo has been populated to sort them. """ + self.aoEnterInfo = sorted(self.aoEnterInfo, key = lambda oTrans: oTrans.iRevision); + self.aoLeaveInfo = sorted(self.aoLeaveInfo, key = lambda oTrans: oTrans.iRevision, reverse = True); + return self; + +class ReportPeriodSetWithTotalBase(ReportPeriodSetBase): + """ In addition to the cHits, we also have a total to relate it too. """ + def __init__(self, sIdAttr): + ReportPeriodSetBase.__init__(self, sIdAttr); + self.dcTotalPerId = {}; # Sum total per subject ID (key). + self.cTotal = 0; # Sum number of total in all periods and all reasons. + self.cMaxTotal = 0; # Max total in a row. + self.cMinTotal = 0; # Min total in a row. + self.uMaxPct = 0; # Max percentage in a row (100 = 100%). + + def _doStatsForPeriod(self, oPeriod): + assert isinstance(oPeriod, ReportPeriodWithTotalBase); + super(ReportPeriodSetWithTotalBase, self)._doStatsForPeriod(oPeriod); + self.cTotal += oPeriod.cTotal; + if oPeriod.cMaxTotal > self.cMaxTotal: + self.cMaxTotal = oPeriod.cMaxTotal; + if oPeriod.cMinTotal < self.cMinTotal: + self.cMinTotal = oPeriod.cMinTotal; + + if oPeriod.uMaxPct > self.uMaxPct: + self.uMaxPct = oPeriod.uMaxPct; + + def recalcStats(self): + self.dcTotalPerId = {}; + self.cTotal = 0; + self.cMaxTotal = 0; + self.cMinTotal = 0; + self.uMaxPct = 0; + super(ReportPeriodSetWithTotalBase, self).recalcStats(); + + def deleteKey(self, idKey): + self.cTotal -= self.dcTotalPerId[idKey]; + del self.dcTotalPerId[idKey]; + super(ReportPeriodSetWithTotalBase, self).deleteKey(idKey); + +class ReportFailureReasonSet(ReportPeriodSetBase): + """ What ReportLazyModel.getFailureReasons returns. """ + def __init__(self): + ReportPeriodSetBase.__init__(self, 'idFailureReason'); + + + +class ReportLazyModel(ReportModelBase): # pylint: disable=too-few-public-methods + """ + The 'lazy bird' report model class. + + We may want to have several classes, maybe one for each report even. But, + I'm thinking that's a bit overkill so we'll start with this and split it + if/when it becomes necessary. + """ + + kdsStatusSimplificationMap = { + ReportModelBase.ksTestStatus_Running: ReportModelBase.ksTestStatus_Running, + ReportModelBase.ksTestStatus_Success: ReportModelBase.ksTestStatus_Success, + ReportModelBase.ksTestStatus_Skipped: ReportModelBase.ksTestStatus_Skipped, + ReportModelBase.ksTestStatus_BadTestBox: ReportModelBase.ksTestStatus_Skipped, + ReportModelBase.ksTestStatus_Aborted: ReportModelBase.ksTestStatus_Skipped, + ReportModelBase.ksTestStatus_Failure: ReportModelBase.ksTestStatus_Failure, + ReportModelBase.ksTestStatus_TimedOut: ReportModelBase.ksTestStatus_Failure, + ReportModelBase.ksTestStatus_Rebooted: ReportModelBase.ksTestStatus_Failure, + }; + + def getSuccessRates(self): + """ + Gets the success rates of the subject in the specified period. + + Returns an array of data per period (0 is the oldes, self.cPeriods-1 is + the latest) where each entry is a status (TestStatus_T) dictionary with + the number of occurences of each final status (i.e. not running). + """ + + sBaseQuery = 'SELECT TestSets.enmStatus, COUNT(TestSets.idTestSet)\n' \ + 'FROM TestSets\n' \ + + self.oFilter.getTableJoins(); + for sTable in self.getExtraSubjectTables(): + sBaseQuery = sBaseQuery[:-1] + ',\n ' + sTable + '\n'; + sBaseQuery += 'WHERE enmStatus <> \'running\'\n' \ + + self.oFilter.getWhereConditions() \ + + self.getExtraSubjectWhereExpr(); + + adPeriods = []; + for iPeriod in xrange(self.cPeriods): + self._oDb.execute(sBaseQuery + self.getExtraWhereExprForPeriod(iPeriod) + 'GROUP BY enmStatus\n'); + + dRet = \ + { + self.ksTestStatus_Skipped: 0, + self.ksTestStatus_Failure: 0, + self.ksTestStatus_Success: 0, + }; + + for aoRow in self._oDb.fetchAll(): + sKey = self.kdsStatusSimplificationMap[aoRow[0]] + if sKey in dRet: + dRet[sKey] += aoRow[1]; + else: + dRet[sKey] = aoRow[1]; + + assert len(dRet) == 3; + + adPeriods.insert(0, dRet); + + return adPeriods; + + + def getFailureReasons(self): + """ + Gets the failure reasons of the subject in the specified period. + + Returns a ReportFailureReasonSet instance. + """ + + oFailureReasonLogic = FailureReasonLogic(self._oDb); + + # + # Create a temporary table + # + sTsNow = 'CURRENT_TIMESTAMP' if self.tsNow is None else self._oDb.formatBindArgs('%s::TIMESTAMP', (self.tsNow,)); + sTsFirst = '(%s - interval \'%s hours\')' \ + % (sTsNow, self.cHoursPerPeriod * self.cPeriods,); + sQuery = 'CREATE TEMPORARY TABLE TmpReasons ON COMMIT DROP AS\n' \ + 'SELECT TestResultFailures.idFailureReason AS idFailureReason,\n' \ + ' TestResultFailures.idTestResult AS idTestResult,\n' \ + ' TestSets.idTestSet AS idTestSet,\n' \ + ' TestSets.tsDone AS tsDone,\n' \ + ' TestSets.tsCreated AS tsCreated,\n' \ + ' TestSets.idBuild AS idBuild\n' \ + 'FROM TestResultFailures,\n' \ + ' TestResults,\n' \ + ' TestSets\n' \ + + self.oFilter.getTableJoins(dOmitTables = {'TestResults': True, 'TestResultFailures': True}); + for sTable in self.getExtraSubjectTables(): + if sTable not in [ 'TestResults', 'TestResultFailures' ] and not self.oFilter.isJoiningWithTable(sTable): + sQuery = sQuery[:-1] + ',\n ' + sTable + '\n'; + sQuery += 'WHERE TestResultFailures.idTestResult = TestResults.idTestResult\n' \ + ' AND TestResultFailures.tsExpire = \'infinity\'::TIMESTAMP\n' \ + ' AND TestResultFailures.tsEffective >= ' + sTsFirst + '\n' \ + ' AND TestResults.enmStatus <> \'running\'\n' \ + ' AND TestResults.enmStatus <> \'success\'\n' \ + ' AND TestResults.tsCreated >= ' + sTsFirst + '\n' \ + ' AND TestResults.tsCreated < ' + sTsNow + '\n' \ + ' AND TestResults.idTestSet = TestSets.idTestSet\n' \ + ' AND TestSets.tsDone >= ' + sTsFirst + '\n' \ + ' AND TestSets.tsDone < ' + sTsNow + '\n' \ + + self.oFilter.getWhereConditions() \ + + self.getExtraSubjectWhereExpr(); + self._oDb.execute(sQuery); + self._oDb.execute('SELECT idFailureReason FROM TmpReasons;'); + + # + # Retrieve the period results. + # + oSet = ReportFailureReasonSet(); + for iPeriod in xrange(self.cPeriods): + self._oDb.execute('SELECT idFailureReason,\n' + ' COUNT(idTestResult),\n' + ' MIN(tsDone),\n' + ' MAX(tsDone)\n' + 'FROM TmpReasons\n' + 'WHERE TRUE\n' + + self.getExtraWhereExprForPeriod(iPeriod).replace('TestSets.', '') + + 'GROUP BY idFailureReason\n'); + aaoRows = self._oDb.fetchAll() + + oPeriod = ReportFailureReasonPeriod(oSet, iPeriod, self.getStraightPeriodDesc(iPeriod), + self.getPeriodStart(iPeriod), self.getPeriodEnd(iPeriod)); + + for aoRow in aaoRows: + oReason = oFailureReasonLogic.cachedLookup(aoRow[0]); + oPeriodRow = ReportFailureReasonRow(aoRow, oReason); + oPeriod.appendRow(oPeriodRow, oReason.idFailureReason, oReason); + + # Count how many test sets we've got without any reason associated with them. + self._oDb.execute('SELECT COUNT(TestSets.idTestSet)\n' + 'FROM TestSets\n' + ' LEFT OUTER JOIN TestResultFailures\n' + ' ON TestSets.idTestSet = TestResultFailures.idTestSet\n' + ' AND TestResultFailures.tsEffective = \'infinity\'::TIMESTAMP\n' + 'WHERE TestSets.enmStatus <> \'running\'\n' + ' AND TestSets.enmStatus <> \'success\'\n' + + self.getExtraWhereExprForPeriod(iPeriod) + + ' AND TestResultFailures.idTestSet IS NULL\n'); + oPeriod.cWithoutReason = self._oDb.fetchOne()[0]; + + oSet.appendPeriod(oPeriod); + + + # + # For reasons entering after the first period, look up the build and + # test set it first occured with. + # + oSet.finalizePass1(); + + for iPeriod in xrange(1, self.cPeriods): + oPeriod = oSet.aoPeriods[iPeriod]; + for oReason in oPeriod.dFirst.values(): + oSet.aoEnterInfo.append(self._getEdgeFailureReasonOccurence(oReason, iPeriod, fEnter = True)); + + # Ditto for reasons leaving before the last. + for iPeriod in xrange(self.cPeriods - 1): + oPeriod = oSet.aoPeriods[iPeriod]; + for oReason in oPeriod.dLast.values(): + oSet.aoLeaveInfo.append(self._getEdgeFailureReasonOccurence(oReason, iPeriod, fEnter = False)); + + oSet.finalizePass2(); + + self._oDb.execute('DROP TABLE TmpReasons\n'); + return oSet; + + + def _getEdgeFailureReasonOccurence(self, oReason, iPeriod, fEnter = True): + """ + Helper for the failure reason report that finds the oldest or newest build + (SVN rev) and test set (start time) it occured with. + + If fEnter is set the oldest occurence is return, if fEnter clear the newest + is is returned. + + Returns ReportFailureReasonTransient instant. + + """ + + + sSorting = 'ASC' if fEnter else 'DESC'; + self._oDb.execute('SELECT TmpReasons.idTestResult,\n' + ' TmpReasons.idTestSet,\n' + ' TmpReasons.tsDone,\n' + ' TmpReasons.idBuild,\n' + ' Builds.iRevision,\n' + ' BuildCategories.sRepository\n' + 'FROM TmpReasons,\n' + ' Builds,\n' + ' BuildCategories\n' + 'WHERE TmpReasons.idFailureReason = %s\n' + ' AND TmpReasons.idBuild = Builds.idBuild\n' + ' AND Builds.tsExpire > TmpReasons.tsCreated\n' + ' AND Builds.tsEffective <= TmpReasons.tsCreated\n' + ' AND Builds.idBuildCategory = BuildCategories.idBuildCategory\n' + 'ORDER BY Builds.iRevision ' + sSorting + ',\n' + ' TmpReasons.tsCreated ' + sSorting + '\n' + 'LIMIT 1\n' + , ( oReason.idFailureReason, )); + aoRow = self._oDb.fetchOne(); + if aoRow is None: + return ReportFailureReasonTransient(-1, -1, 'internal-error', -1, -1, self._oDb.getCurrentTimestamp(), + iPeriod, fEnter, oReason); + return ReportFailureReasonTransient(idBuild = aoRow[3], iRevision = aoRow[4], sRepository = aoRow[5], + idTestSet = aoRow[1], idTestResult = aoRow[0], tsDone = aoRow[2], + iPeriod = iPeriod, fEnter = fEnter, oReason = oReason); + + + def getTestCaseFailures(self): + """ + Gets the test case failures of the subject in the specified period. + + Returns a ReportPeriodSetWithTotalBase instance. + + """ + return self._getSimpleFailures('idTestCase', TestCaseLogic); + + + def getTestCaseVariationFailures(self): + """ + Gets the test case failures of the subject in the specified period. + + Returns a ReportPeriodSetWithTotalBase instance. + + """ + return self._getSimpleFailures('idTestCaseArgs', TestCaseArgsLogic); + + + def getTestBoxFailures(self): + """ + Gets the test box failures of the subject in the specified period. + + Returns a ReportPeriodSetWithTotalBase instance. + + """ + return self._getSimpleFailures('idTestBox', TestBoxLogic); + + + def _getSimpleFailures(self, sIdColumn, oCacheLogicType, sIdAttr = None): + """ + Gets the test box failures of the subject in the specified period. + + Returns a ReportPeriodSetWithTotalBase instance. + + """ + + oLogic = oCacheLogicType(self._oDb); + oSet = ReportPeriodSetWithTotalBase(sIdColumn if sIdAttr is None else sIdAttr); + + # Construct base query. + sBaseQuery = 'SELECT TestSets.' + sIdColumn + ',\n' \ + ' COUNT(CASE WHEN TestSets.enmStatus >= \'failure\' THEN 1 END),\n' \ + ' MIN(TestSets.tsDone),\n' \ + ' MAX(TestSets.tsDone),\n' \ + ' COUNT(TestSets.idTestResult)\n' \ + 'FROM TestSets\n' \ + + self.oFilter.getTableJoins(); + for sTable in self.getExtraSubjectTables(): + sBaseQuery = sBaseQuery[:-1] + ',\n ' + sTable + '\n'; + sBaseQuery += 'WHERE TRUE\n' \ + + self.oFilter.getWhereConditions() \ + + self.getExtraSubjectWhereExpr() + '\n'; + + # Retrieve the period results. + for iPeriod in xrange(self.cPeriods): + self._oDb.execute(sBaseQuery + self.getExtraWhereExprForPeriod(iPeriod) + 'GROUP BY TestSets.' + sIdColumn + '\n'); + aaoRows = self._oDb.fetchAll() + + oPeriod = ReportPeriodWithTotalBase(oSet, iPeriod, self.getStraightPeriodDesc(iPeriod), + self.getPeriodStart(iPeriod), self.getPeriodEnd(iPeriod)); + + for aoRow in aaoRows: + oSubject = oLogic.cachedLookup(aoRow[0]); + oPeriodRow = ReportHitRowWithTotalBase(aoRow[0], oSubject, aoRow[1], aoRow[4], aoRow[2], aoRow[3]); + oPeriod.appendRow(oPeriodRow, aoRow[0], oSubject); + + oSet.appendPeriod(oPeriod); + oSet.pruneRowsWithZeroSumHits(); + + + + # + # For reasons entering after the first period, look up the build and + # test set it first occured with. + # + oSet.finalizePass1(); + + for iPeriod in xrange(1, self.cPeriods): + oPeriod = oSet.aoPeriods[iPeriod]; + for idSubject, oSubject in oPeriod.dFirst.items(): + oSet.aoEnterInfo.append(self._getEdgeSimpleFailureOccurence(idSubject, sIdColumn, oSubject, + iPeriod, fEnter = True)); + + # Ditto for reasons leaving before the last. + for iPeriod in xrange(self.cPeriods - 1): + oPeriod = oSet.aoPeriods[iPeriod]; + for idSubject, oSubject in oPeriod.dLast.items(): + oSet.aoLeaveInfo.append(self._getEdgeSimpleFailureOccurence(idSubject, sIdColumn, oSubject, + iPeriod, fEnter = False)); + + oSet.finalizePass2(); + + return oSet; + + def _getEdgeSimpleFailureOccurence(self, idSubject, sIdColumn, oSubject, iPeriod, fEnter = True): + """ + Helper for the failure reason report that finds the oldest or newest build + (SVN rev) and test set (start time) it occured with. + + If fEnter is set the oldest occurence is return, if fEnter clear the newest + is is returned. + + Returns ReportTransientBase instant. + + """ + sSorting = 'ASC' if fEnter else 'DESC'; + sQuery = 'SELECT TestSets.idTestResult,\n' \ + ' TestSets.idTestSet,\n' \ + ' TestSets.tsDone,\n' \ + ' TestSets.idBuild,\n' \ + ' Builds.iRevision,\n' \ + ' BuildCategories.sRepository\n' \ + 'FROM TestSets\n' \ + + self.oFilter.getTableJoins(dOmitTables = {'Builds': True, 'BuildCategories': True}); + sQuery = sQuery[:-1] + ',\n' \ + ' Builds,\n' \ + ' BuildCategories\n'; + for sTable in self.getExtraSubjectTables(): + if sTable not in [ 'Builds', 'BuildCategories' ] and not self.oFilter.isJoiningWithTable(sTable): + sQuery = sQuery[:-1] + ',\n ' + sTable + '\n'; + sQuery += 'WHERE TestSets.' + sIdColumn + ' = ' + str(idSubject) + '\n' \ + ' AND TestSets.idBuild = Builds.idBuild\n' \ + ' AND TestSets.enmStatus >= \'failure\'\n' \ + + self.getExtraWhereExprForPeriod(iPeriod) + \ + ' AND Builds.tsExpire > TestSets.tsCreated\n' \ + ' AND Builds.tsEffective <= TestSets.tsCreated\n' \ + ' AND Builds.idBuildCategory = BuildCategories.idBuildCategory\n' \ + + self.oFilter.getWhereConditions() \ + + self.getExtraSubjectWhereExpr() + '\n' \ + 'ORDER BY Builds.iRevision ' + sSorting + ',\n' \ + ' TestSets.tsCreated ' + sSorting + '\n' \ + 'LIMIT 1\n'; + self._oDb.execute(sQuery); + aoRow = self._oDb.fetchOne(); + if aoRow is None: + return ReportTransientBase(-1, -1, 'internal-error', -1, -1, self._oDb.getCurrentTimestamp(), + iPeriod, fEnter, idSubject, oSubject); + return ReportTransientBase(idBuild = aoRow[3], iRevision = aoRow[4], sRepository = aoRow[5], + idTestSet = aoRow[1], idTestResult = aoRow[0], tsDone = aoRow[2], + iPeriod = iPeriod, fEnter = fEnter, idSubject = idSubject, oSubject = oSubject); + + def fetchPossibleFilterOptions(self, oFilter, tsNow, sPeriod): + """ + Fetches possible filtering options. + """ + return TestResultLogic(self._oDb).fetchPossibleFilterOptions(oFilter, tsNow, sPeriod, oReportModel = self); + + + +class ReportGraphModel(ReportModelBase): # pylint: disable=too-few-public-methods + """ + Extended report model used when generating the more complicated graphs + detailing results, time elapsed and values over time. + """ + + ## @name Subject ID types. + ## These prefix the values in the aidSubjects array. The prefix is + ## followed by a colon and then a list of string IDs. Following the prefix + ## is one or more string table IDs separated by colons. These are used to + ## drill down the exact test result we're looking for, by matching against + ## TestResult::idStrName (in the db). + ## @{ + ksTypeResult = 'result'; + ksTypeElapsed = 'elapsed'; + ## The last string table ID gives the name of the value. + ksTypeValue = 'value'; + ## List of types. + kasTypes = (ksTypeResult, ksTypeElapsed, ksTypeValue); + ## @} + + class SampleSource(object): + """ A sample source. """ + def __init__(self, sType, aidStrTests, idStrValue): + self.sType = sType; + self.aidStrTests = aidStrTests; + self.idStrValue = idStrValue; + + def getTestResultTables(self): + """ Retrieves the list of TestResults tables to join with.""" + sRet = ''; + for i in range(len(self.aidStrTests)): + sRet += ' TestResults TR%u,\n' % (i,); + return sRet; + + def getTestResultConditions(self): + """ Retrieves the join conditions for the TestResults tables.""" + sRet = ''; + cItems = len(self.aidStrTests); + for i in range(cItems - 1): + sRet += ' AND TR%u.idStrName = %u\n' \ + ' AND TR%u.idTestResultParent = TR%u.idTestResult\n' \ + % ( i, self.aidStrTests[cItems - i - 1], i, i + 1 ); + sRet += ' AND TR%u.idStrName = %u\n' % (cItems - 1, self.aidStrTests[0]); + return sRet; + + class DataSeries(object): + """ A data series. """ + def __init__(self, oCache, idBuildCategory, idTestBox, idTestCase, idTestCaseArgs, iUnit): + _ = oCache; + self.idBuildCategory = idBuildCategory; + self.oBuildCategory = oCache.getBuildCategory(idBuildCategory); + self.idTestBox = idTestBox; + self.oTestBox = oCache.getTestBox(idTestBox); + self.idTestCase = idTestCase; + self.idTestCaseArgs = idTestCaseArgs; + if idTestCase is not None: + self.oTestCase = oCache.getTestCase(idTestCase); + self.oTestCaseArgs = None; + else: + self.oTestCaseArgs = oCache.getTestCaseArgs(idTestCaseArgs); + self.oTestCase = oCache.getTestCase(self.oTestCaseArgs.idTestCase); + self.iUnit = iUnit; + # Six parallel arrays. + self.aiRevisions = []; # The X values. + self.aiValues = []; # The Y values. + self.aiErrorBarBelow = []; # The Y value minimum errorbars, relative to the Y value (positive). + self.aiErrorBarAbove = []; # The Y value maximum errorbars, relative to the Y value (positive). + self.acSamples = []; # The number of samples at this X value. + self.aoRevInfo = []; # VcsRevisionData objects for each revision. Empty/SQL-NULL objects if no info. + + class DataSeriesCollection(object): + """ A collection of data series corresponding to one input sample source. """ + def __init__(self, oSampleSrc, asTests, sValue = None): + self.sType = oSampleSrc.sType; + self.aidStrTests = oSampleSrc.aidStrTests; + self.asTests = list(asTests); + self.idStrValue = oSampleSrc.idStrValue; + self.sValue = sValue; + self.aoSeries = []; + + def addDataSeries(self, oDataSeries): + """ Appends a data series to the collection. """ + self.aoSeries.append(oDataSeries); + return oDataSeries; + + + def __init__(self, oDb, tsNow, cPeriods, cHoursPerPeriod, sSubject, aidSubjects, # pylint: disable=too-many-arguments + aidTestBoxes, aidBuildCats, aidTestCases, fSepTestVars): + assert(sSubject == self.ksSubEverything); # dummy + ReportModelBase.__init__(self, oDb, tsNow, cPeriods, cHoursPerPeriod, sSubject, aidSubjects, oFilter = None); + self.aidTestBoxes = aidTestBoxes; + self.aidBuildCats = aidBuildCats; + self.aidTestCases = aidTestCases; + self.fOnTestCase = not fSepTestVars; # (Separates testcase variations into separate data series.) + self.oCache = DatabaseObjCache(self._oDb, self.tsNow, None, self.cPeriods * self.cHoursPerPeriod); + + + # Quickly validate and convert the subject "IDs". + self.aoLookups = []; + for sCur in self.aidSubjects: + asParts = sCur.split(':'); + if len(asParts) < 2: + raise TMExceptionBase('Invalid graph value "%s"' % (sCur,)); + + sType = asParts[0]; + if sType not in ReportGraphModel.kasTypes: + raise TMExceptionBase('Invalid graph value type "%s" (full: "%s")' % (sType, sCur,)); + + aidStrTests = []; + for sIdStr in asParts[1:]: + try: idStr = int(sIdStr); + except: raise TMExceptionBase('Invalid graph value id "%s" (full: "%s")' % (sIdStr, sCur,)); + if idStr < 0: + raise TMExceptionBase('Invalid graph value id "%u" (full: "%s")' % (idStr, sCur,)); + aidStrTests.append(idStr); + + idStrValue = None; + if sType == ReportGraphModel.ksTypeValue: + idStrValue = aidStrTests.pop(); + self.aoLookups.append(ReportGraphModel.SampleSource(sType, aidStrTests, idStrValue)); + + # done + + + def getExtraWhereExprForTotalPeriod(self, sTimestampField): + """ + Returns additional WHERE expression for getting test sets for the + specified period. It starts with an AND so that it can simply be + appended to the WHERE clause. + """ + return self.getExtraWhereExprForTotalPeriodEx(sTimestampField, sTimestampField, True); + + def getExtraWhereExprForTotalPeriodEx(self, sStartField = 'tsCreated', sEndField = 'tsDone', fLeadingAnd = True): + """ + Returns additional WHERE expression for getting test sets for the + specified period. + """ + if self.tsNow is None: + sNow = 'CURRENT_TIMESTAMP'; + else: + sNow = self._oDb.formatBindArgs('%s::TIMESTAMP', (self.tsNow,)); + + sRet = ' AND %s >= (%s - interval \'%u hours\')\n' \ + ' AND %s <= %s\n' \ + % ( sStartField, sNow, self.cPeriods * self.cHoursPerPeriod, + sEndField, sNow); + + if not fLeadingAnd: + assert sRet[8] == ' ' and sRet[7] == 'D'; + return sRet[9:]; + return sRet; + + def _getEligibleTestSetPeriod(self, sPrefix = 'TestSets.', fLeadingAnd = False): + """ + Returns additional WHERE expression for getting TestSets rows + potentially relevant for the selected period. + """ + if self.tsNow is None: + sNow = 'CURRENT_TIMESTAMP'; + else: + sNow = self._oDb.formatBindArgs('%s::TIMESTAMP', (self.tsNow,)); + + # The 2nd line is a performance hack on TestSets. It nudges postgresql + # into useing the TestSetsCreatedDoneIdx index instead of doing a table + # scan when we look for eligible bits there. + # ASSUMES no relevant test runs longer than 7 days! + sRet = ' AND %stsCreated <= %s\n' \ + ' AND %stsCreated >= (%s - interval \'%u hours\' - interval \'%u days\')\n' \ + ' AND %stsDone >= (%s - interval \'%u hours\')\n' \ + % ( sPrefix, sNow, + sPrefix, sNow, self.cPeriods * self.cHoursPerPeriod, 7, + sPrefix, sNow, self.cPeriods * self.cHoursPerPeriod, ); + + if not fLeadingAnd: + assert sRet[8] == ' ' and sRet[7] == 'D'; + return sRet[9:]; + return sRet; + + + def _getNameStrings(self, aidStrTests): + """ Returns an array of names corresponding to the array of string table entries. """ + return [self.oCache.getTestResultString(idStr) for idStr in aidStrTests]; + + def fetchGraphData(self): + """ returns data """ + sWantedTestCaseId = 'idTestCase' if self.fOnTestCase else 'idTestCaseArgs'; + + aoRet = []; + for oLookup in self.aoLookups: + # + # Set up the result collection. + # + if oLookup.sType == self.ksTypeValue: + oCollection = self.DataSeriesCollection(oLookup, self._getNameStrings(oLookup.aidStrTests), + self.oCache.getTestResultString(oLookup.idStrValue)); + else: + oCollection = self.DataSeriesCollection(oLookup, self._getNameStrings(oLookup.aidStrTests)); + + # + # Construct the query. + # + sQuery = 'SELECT Builds.iRevision,\n' \ + ' TestSets.idBuildCategory,\n' \ + ' TestSets.idTestBox,\n' \ + ' TestSets.' + sWantedTestCaseId + ',\n'; + if oLookup.sType == self.ksTypeValue: + sQuery += ' TestResultValues.iUnit as iUnit,\n' \ + ' MIN(TestResultValues.lValue),\n' \ + ' CAST(ROUND(AVG(TestResultValues.lValue)) AS BIGINT),\n' \ + ' MAX(TestResultValues.lValue),\n' \ + ' COUNT(TestResultValues.lValue)\n'; + elif oLookup.sType == self.ksTypeElapsed: + sQuery += ' %u as iUnit,\n' \ + ' CAST((EXTRACT(EPOCH FROM MIN(TR0.tsElapsed)) * 1000) AS INTEGER),\n' \ + ' CAST((EXTRACT(EPOCH FROM AVG(TR0.tsElapsed)) * 1000) AS INTEGER),\n' \ + ' CAST((EXTRACT(EPOCH FROM MAX(TR0.tsElapsed)) * 1000) AS INTEGER),\n' \ + ' COUNT(TR0.tsElapsed)\n' \ + % (constants.valueunit.MS,); + else: + sQuery += ' %u as iUnit,\n'\ + ' MIN(TR0.cErrors),\n' \ + ' CAST(ROUND(AVG(TR0.cErrors)) AS INTEGER),\n' \ + ' MAX(TR0.cErrors),\n' \ + ' COUNT(TR0.cErrors)\n' \ + % (constants.valueunit.OCCURRENCES,); + + if oLookup.sType == self.ksTypeValue: + sQuery += 'FROM TestResultValues,\n'; + sQuery += ' TestSets,\n' + sQuery += oLookup.getTestResultTables(); + else: + sQuery += 'FROM ' + oLookup.getTestResultTables().lstrip(); + sQuery += ' TestSets,\n'; + sQuery += ' Builds\n'; + + if oLookup.sType == self.ksTypeValue: + sQuery += 'WHERE TestResultValues.idStrName = %u\n' % ( oLookup.idStrValue, ); + sQuery += self.getExtraWhereExprForTotalPeriod('TestResultValues.tsCreated'); + sQuery += ' AND TestResultValues.idTestSet = TestSets.idTestSet\n'; + sQuery += self._getEligibleTestSetPeriod(fLeadingAnd = True); + else: + sQuery += 'WHERE ' + (self.getExtraWhereExprForTotalPeriod('TR0.tsCreated').lstrip()[4:]).lstrip(); + sQuery += ' AND TR0.idTestSet = TestSets.idTestSet\n'; + + if len(self.aidTestBoxes) == 1: + sQuery += ' AND TestSets.idTestBox = %u\n' % (self.aidTestBoxes[0],); + elif self.aidTestBoxes: + sQuery += ' AND TestSets.idTestBox IN (' + ','.join([str(i) for i in self.aidTestBoxes]) + ')\n'; + + if len(self.aidBuildCats) == 1: + sQuery += ' AND TestSets.idBuildCategory = %u\n' % (self.aidBuildCats[0],); + elif self.aidBuildCats: + sQuery += ' AND TestSets.idBuildCategory IN (' + ','.join([str(i) for i in self.aidBuildCats]) + ')\n'; + + if len(self.aidTestCases) == 1: + sQuery += ' AND TestSets.idTestCase = %u\n' % (self.aidTestCases[0],); + elif self.aidTestCases: + sQuery += ' AND TestSets.idTestCase IN (' + ','.join([str(i) for i in self.aidTestCases]) + ')\n'; + + if oLookup.sType == self.ksTypeElapsed: + sQuery += ' AND TestSets.enmStatus = \'%s\'::TestStatus_T\n' % (self.ksTestStatus_Success,); + + if oLookup.sType == self.ksTypeValue: + sQuery += ' AND TestResultValues.idTestResult = TR0.idTestResult\n' + sQuery += self.getExtraWhereExprForTotalPeriod('TR0.tsCreated'); # For better index matching in some cases. + + if oLookup.sType != self.ksTypeResult: + sQuery += ' AND TR0.enmStatus = \'%s\'::TestStatus_T\n' % (self.ksTestStatus_Success,); + + sQuery += oLookup.getTestResultConditions(); + sQuery += ' AND TestSets.idBuild = Builds.idBuild\n'; + + sQuery += 'GROUP BY TestSets.idBuildCategory,\n' \ + ' TestSets.idTestBox,\n' \ + ' TestSets.' + sWantedTestCaseId + ',\n' \ + ' iUnit,\n' \ + ' Builds.iRevision\n'; + sQuery += 'ORDER BY TestSets.idBuildCategory,\n' \ + ' TestSets.idTestBox,\n' \ + ' TestSets.' + sWantedTestCaseId + ',\n' \ + ' iUnit,\n' \ + ' Builds.iRevision\n'; + + # + # Execute it and collect the result. + # + sCurRepository = None; + dRevisions = {}; + oLastSeries = None; + idLastBuildCat = -1; + idLastTestBox = -1; + idLastTestCase = -1; + iLastUnit = -1; + self._oDb.execute(sQuery); + for aoRow in self._oDb.fetchAll(): # Fetching all here so we can make cache queries below. + if aoRow[1] != idLastBuildCat \ + or aoRow[2] != idLastTestBox \ + or aoRow[3] != idLastTestCase \ + or aoRow[4] != iLastUnit: + idLastBuildCat = aoRow[1]; + idLastTestBox = aoRow[2]; + idLastTestCase = aoRow[3]; + iLastUnit = aoRow[4]; + if self.fOnTestCase: + oLastSeries = self.DataSeries(self.oCache, idLastBuildCat, idLastTestBox, + idLastTestCase, None, iLastUnit); + else: + oLastSeries = self.DataSeries(self.oCache, idLastBuildCat, idLastTestBox, + None, idLastTestCase, iLastUnit); + oCollection.addDataSeries(oLastSeries); + if oLastSeries.oBuildCategory.sRepository != sCurRepository: + if sCurRepository is not None: + self.oCache.preloadVcsRevInfo(sCurRepository, dRevisions.keys()); + sCurRepository = oLastSeries.oBuildCategory.sRepository + dRevisions = {}; + oLastSeries.aiRevisions.append(aoRow[0]); + oLastSeries.aiValues.append(aoRow[6]); + oLastSeries.aiErrorBarBelow.append(aoRow[6] - aoRow[5]); + oLastSeries.aiErrorBarAbove.append(aoRow[7] - aoRow[6]); + oLastSeries.acSamples.append(aoRow[8]); + dRevisions[aoRow[0]] = 1; + + if sCurRepository is not None: + self.oCache.preloadVcsRevInfo(sCurRepository, dRevisions.keys()); + del dRevisions; + + # + # Look up the VCS revision details. + # + for oSeries in oCollection.aoSeries: + for iRevision in oSeries.aiRevisions: + oSeries.aoRevInfo.append(self.oCache.getVcsRevInfo(sCurRepository, iRevision)); + aoRet.append(oCollection); + + return aoRet; + + def getEligibleTestBoxes(self): + """ + Returns a list of TestBoxData objects with eligible testboxes for + the total period of time defined for this graph. + """ + + # Taking the simple way out now, getting all active testboxes at the + # time without filtering out on sample sources. + + # 1. Collect the relevant testbox generation IDs. + self._oDb.execute('SELECT DISTINCT idTestBox, idGenTestBox\n' + 'FROM TestSets\n' + 'WHERE ' + self._getEligibleTestSetPeriod(fLeadingAnd = False) + + 'ORDER BY idTestBox, idGenTestBox DESC'); + idPrevTestBox = -1; + asIdGenTestBoxes = []; + for _ in range(self._oDb.getRowCount()): + aoRow = self._oDb.fetchOne(); + if aoRow[0] != idPrevTestBox: + idPrevTestBox = aoRow[0]; + asIdGenTestBoxes.append(str(aoRow[1])); + + # 2. Query all the testbox data in one go. + aoRet = []; + if asIdGenTestBoxes: + self._oDb.execute('SELECT *\n' + 'FROM TestBoxesWithStrings\n' + 'WHERE idGenTestBox IN (' + ','.join(asIdGenTestBoxes) + ')\n' + 'ORDER BY sName'); + for _ in range(self._oDb.getRowCount()): + aoRet.append(TestBoxData().initFromDbRow(self._oDb.fetchOne())); + + return aoRet; + + def getEligibleBuildCategories(self): + """ + Returns a list of BuildCategoryData objects with eligible build + categories for the total period of time defined for this graph. In + addition it will add any currently selected categories that aren't + really relevant to the period, just to simplify the WUI code. + + """ + + # Taking the simple way out now, getting all used build cat without + # any testbox or testcase filtering. + + sSelectedBuildCats = ''; + if self.aidBuildCats: + sSelectedBuildCats = ' OR idBuildCategory IN (' + ','.join([str(i) for i in self.aidBuildCats]) + ')\n'; + + self._oDb.execute('SELECT DISTINCT *\n' + 'FROM BuildCategories\n' + 'WHERE idBuildCategory IN (\n' + ' SELECT DISTINCT idBuildCategory\n' + ' FROM TestSets\n' + ' WHERE ' + self._getEligibleTestSetPeriod(fLeadingAnd = False) + + ')\n' + + sSelectedBuildCats + + 'ORDER BY sProduct,\n' + ' sBranch,\n' + ' asOsArches,\n' + ' sType\n'); + aoRet = []; + for _ in range(self._oDb.getRowCount()): + aoRet.append(BuildCategoryData().initFromDbRow(self._oDb.fetchOne())); + + return aoRet; + diff --git a/src/VBox/ValidationKit/testmanager/core/restdispatcher.py b/src/VBox/ValidationKit/testmanager/core/restdispatcher.py new file mode 100755 index 00000000..75a1aa7c --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/restdispatcher.py @@ -0,0 +1,455 @@ +# -*- coding: utf-8 -*- +# $Id: restdispatcher.py $ + +""" +Test Manager Core - REST cgi handler. +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Standard python imports. +import os; +import sys; + +# Validation Kit imports. +#from common import constants; +from common import utils; +from testmanager import config; +#from testmanager.core import coreconsts; +from testmanager.core.db import TMDatabaseConnection; +from testmanager.core.base import TMExceptionBase, ModelDataBase; + +# Python 3 hacks: +if sys.version_info[0] >= 3: + long = int; # pylint: disable=redefined-builtin,invalid-name + + +# +# Exceptions +# + +class RestDispException(TMExceptionBase): + """ + Exception class for the REST dispatcher. + """ + def __init__(self, sMsg, iStatus): + TMExceptionBase.__init__(self, sMsg); + self.iStatus = iStatus; + +# 400 +class RestDispException400(RestDispException): + """ A 400 error """ + def __init__(self, sMsg): + RestDispException.__init__(self, sMsg, 400); + +class RestUnknownParameters(RestDispException400): + """ Unknown parameter(s). """ + pass; # pylint: disable=unnecessary-pass + +# 404 +class RestDispException404(RestDispException): + """ A 404 error """ + def __init__(self, sMsg): + RestDispException.__init__(self, sMsg, 404); + +class RestBadPathException(RestDispException404): + """ We've got a bad path. """ + pass; # pylint: disable=unnecessary-pass + +class RestBadParameter(RestDispException404): + """ Bad parameter. """ + pass; # pylint: disable=unnecessary-pass + +class RestMissingParameter(RestDispException404): + """ Missing parameter. """ + pass; # pylint: disable=unnecessary-pass + + + +class RestMain(object): # pylint: disable=too-few-public-methods + """ + REST main dispatcher class. + """ + + ksParam_sPath = 'sPath'; + + + def __init__(self, oSrvGlue): + self._oSrvGlue = oSrvGlue; + self._oDb = TMDatabaseConnection(oSrvGlue.dprint); + self._iFirstHandlerPath = 0; + self._iNextHandlerPath = 0; + self._sPath = None; # _getStandardParams / dispatchRequest sets this later on. + self._asPath = None; # _getStandardParams / dispatchRequest sets this later on. + self._sMethod = None; # _getStandardParams / dispatchRequest sets this later on. + self._dParams = None; # _getStandardParams / dispatchRequest sets this later on. + self._asCheckedParams = []; + self._dGetTree = { + 'vcs': { + 'changelog': self._handleVcsChangelog_Get, + 'bugreferences': self._handleVcsBugReferences_Get, + }, + }; + self._dMethodTrees = { + 'GET': self._dGetTree, + } + + # + # Helpers. + # + + def _getStringParam(self, sName, asValidValues = None, fStrip = False, sDefValue = None): + """ + Gets a string parameter (stripped). + + Raises exception if not found and no default is provided, or if the + value isn't found in asValidValues. + """ + if sName not in self._dParams: + if sDefValue is None: + raise RestMissingParameter('%s parameter %s is missing' % (self._sPath, sName)); + return sDefValue; + sValue = self._dParams[sName]; + if isinstance(sValue, list): + if len(sValue) == 1: + sValue = sValue[0]; + else: + raise RestBadParameter('%s parameter %s value is not a string but list: %s' + % (self._sPath, sName, sValue)); + if fStrip: + sValue = sValue.strip(); + + if sName not in self._asCheckedParams: + self._asCheckedParams.append(sName); + + if asValidValues is not None and sValue not in asValidValues: + raise RestBadParameter('%s parameter %s value "%s" not in %s ' + % (self._sPath, sName, sValue, asValidValues)); + return sValue; + + def _getBoolParam(self, sName, fDefValue = None): + """ + Gets a boolean parameter. + + Raises exception if not found and no default is provided, or if not a + valid boolean. + """ + sValue = self._getStringParam(sName, [ 'True', 'true', '1', 'False', 'false', '0'], sDefValue = str(fDefValue)); + return sValue in ('True', 'true', '1',); + + def _getIntParam(self, sName, iMin = None, iMax = None): + """ + Gets a string parameter. + Raises exception if not found, not a valid integer, or if the value + isn't in the range defined by iMin and iMax. + """ + sValue = self._getStringParam(sName); + try: + iValue = int(sValue, 0); + except: + raise RestBadParameter('%s parameter %s value "%s" cannot be convert to an integer' + % (self._sPath, sName, sValue)); + + if (iMin is not None and iValue < iMin) \ + or (iMax is not None and iValue > iMax): + raise RestBadParameter('%s parameter %s value %d is out of range [%s..%s]' + % (self._sPath, sName, iValue, iMin, iMax)); + return iValue; + + def _getLongParam(self, sName, lMin = None, lMax = None, lDefValue = None): + """ + Gets a string parameter. + Raises exception if not found, not a valid long integer, or if the value + isn't in the range defined by lMin and lMax. + """ + sValue = self._getStringParam(sName, sDefValue = (str(lDefValue) if lDefValue is not None else None)); + try: + lValue = long(sValue, 0); + except Exception as oXcpt: + raise RestBadParameter('%s parameter %s value "%s" cannot be convert to an integer (%s)' + % (self._sPath, sName, sValue, oXcpt)); + + if (lMin is not None and lValue < lMin) \ + or (lMax is not None and lValue > lMax): + raise RestBadParameter('%s parameter %s value %d is out of range [%s..%s]' + % (self._sPath, sName, lValue, lMin, lMax)); + return lValue; + + def _checkForUnknownParameters(self): + """ + Check if we've handled all parameters, raises exception if anything + unknown was found. + """ + + if len(self._asCheckedParams) != len(self._dParams): + sUnknownParams = ''; + for sKey in self._dParams: + if sKey not in self._asCheckedParams: + sUnknownParams += ' ' + sKey + '=' + self._dParams[sKey]; + raise RestUnknownParameters('Unknown parameters: ' + sUnknownParams); + + return True; + + def writeToMainLog(self, oTestSet, sText, fIgnoreSizeCheck = False): + """ Writes the text to the main log file. """ + + # Calc the file name and open the file. + sFile = os.path.join(config.g_ksFileAreaRootDir, oTestSet.sBaseFilename + '-main.log'); + if not os.path.exists(os.path.dirname(sFile)): + os.makedirs(os.path.dirname(sFile), 0o755); + + with open(sFile, 'ab') as oFile: + # Check the size. + fSizeOk = True; + if not fIgnoreSizeCheck: + oStat = os.fstat(oFile.fileno()); + fSizeOk = oStat.st_size / (1024 * 1024) < config.g_kcMbMaxMainLog; + + # Write the text. + if fSizeOk: + if sys.version_info[0] >= 3: + oFile.write(bytes(sText, 'utf-8')); + else: + oFile.write(sText); + + return fSizeOk; + + def _getNextPathElementString(self, sName, oDefault = None): + """ + Gets the next handler specific path element. + Returns unprocessed string. + Throws exception + """ + i = self._iNextHandlerPath; + if i < len(self._asPath): + self._iNextHandlerPath = i + 1; + return self._asPath[i]; + if oDefault is None: + raise RestBadPathException('Requires a "%s" element after "%s"' % (sName, self._sPath,)); + return oDefault; + + def _getNextPathElementInt(self, sName, iDefault = None, iMin = None, iMax = None): + """ + Gets the next handle specific path element as an integer. + Returns integer value. + Throws exception if not found or not a valid integer. + """ + sValue = self._getNextPathElementString(sName, oDefault = iDefault); + try: + iValue = int(sValue); + except: + raise RestBadPathException('Not an integer "%s" (%s)' % (sValue, sName,)); + if iMin is not None and iValue < iMin: + raise RestBadPathException('Integer "%s" value (%s) is too small, min %s' % (sValue, sName, iMin)); + if iMax is not None and iValue > iMax: + raise RestBadPathException('Integer "%s" value (%s) is too large, max %s' % (sValue, sName, iMax)); + return iValue; + + def _getNextPathElementLong(self, sName, iDefault = None, iMin = None, iMax = None): + """ + Gets the next handle specific path element as a long integer. + Returns integer value. + Throws exception if not found or not a valid integer. + """ + sValue = self._getNextPathElementString(sName, oDefault = iDefault); + try: + iValue = long(sValue); + except: + raise RestBadPathException('Not an integer "%s" (%s)' % (sValue, sName,)); + if iMin is not None and iValue < iMin: + raise RestBadPathException('Integer "%s" value (%s) is too small, min %s' % (sValue, sName, iMin)); + if iMax is not None and iValue > iMax: + raise RestBadPathException('Integer "%s" value (%s) is too large, max %s' % (sValue, sName, iMax)); + return iValue; + + def _checkNoMorePathElements(self): + """ + Checks that there are no more path elements. + Throws exception if there are. + """ + i = self._iNextHandlerPath; + if i < len(self._asPath): + raise RestBadPathException('Unknown subpath "%s" below "%s"' % + ('/'.join(self._asPath[i:]), '/'.join(self._asPath[:i]),)); + return True; + + def _doneParsingArguments(self): + """ + Checks that there are no more path elements or unhandled parameters. + Throws exception if there are. + """ + self._checkNoMorePathElements(); + self._checkForUnknownParameters(); + return True; + + def _dataArrayToJsonReply(self, aoData, sName = 'aoData', dExtraFields = None, iStatus = 200): + """ + Converts aoData into an array objects + return True. + """ + self._oSrvGlue.setContentType('application/json'); + self._oSrvGlue.setStatus(iStatus); + self._oSrvGlue.write(u'{\n'); + if dExtraFields: + for sKey in dExtraFields: + self._oSrvGlue.write(u' "%s": %s,\n' % (sKey, ModelDataBase.genericToJson(dExtraFields[sKey]),)); + self._oSrvGlue.write(u' "c%s": %u,\n' % (sName[2:],len(aoData),)); + self._oSrvGlue.write(u' "%s": [\n' % (sName,)); + for i, oData in enumerate(aoData): + if i > 0: + self._oSrvGlue.write(u',\n'); + self._oSrvGlue.write(ModelDataBase.genericToJson(oData)); + self._oSrvGlue.write(u' ]\n'); + ## @todo if config.g_kfWebUiSqlDebug: + self._oSrvGlue.write(u'}\n'); + self._oSrvGlue.flush(); + return True; + + + # + # Handlers. + # + + def _handleVcsChangelog_Get(self): + """ GET /vcs/changelog/{sRepository}/{iStartRev}[/{cEntriesBack}] """ + # Parse arguments + sRepository = self._getNextPathElementString('sRepository'); + iStartRev = self._getNextPathElementInt('iStartRev', iMin = 0); + cEntriesBack = self._getNextPathElementInt('cEntriesBack', iDefault = 32, iMin = 0, iMax = 8192); + self._checkNoMorePathElements(); + self._checkForUnknownParameters(); + + # Execute it. + from testmanager.core.vcsrevisions import VcsRevisionLogic; + oLogic = VcsRevisionLogic(self._oDb); + return self._dataArrayToJsonReply(oLogic.fetchTimeline(sRepository, iStartRev, cEntriesBack), 'aoCommits', + { 'sTracChangesetUrlFmt': + config.g_ksTracChangsetUrlFmt.replace('%(sRepository)s', sRepository), } ); + + def _handleVcsBugReferences_Get(self): + """ GET /vcs/bugreferences/{sTrackerId}/{lBugId} """ + # Parse arguments + sTrackerId = self._getNextPathElementString('sTrackerId'); + lBugId = self._getNextPathElementLong('lBugId', iMin = 0); + self._checkNoMorePathElements(); + self._checkForUnknownParameters(); + + # Execute it. + from testmanager.core.vcsbugreference import VcsBugReferenceLogic; + oLogic = VcsBugReferenceLogic(self._oDb); + oLogic.fetchForBug(sTrackerId, lBugId) + return self._dataArrayToJsonReply(oLogic.fetchForBug(sTrackerId, lBugId), 'aoCommits', + { 'sTracChangesetUrlFmt': config.g_ksTracChangsetUrlFmt, } ); + + + # + # Dispatching. + # + + def _dispatchRequestCommon(self): + """ + Dispatches the incoming request after have gotten the path and parameters. + + Will raise RestDispException on failure. + """ + + # + # Split up the path. + # + asPath = self._sPath.split('/'); + self._asPath = asPath; + + # + # Get the method and the corresponding handler tree. + # + try: + sMethod = self._oSrvGlue.getMethod(); + except Exception as oXcpt: + raise RestDispException('Error retriving request method: %s' % (oXcpt,), 400); + self._sMethod = sMethod; + + try: + dTree = self._dMethodTrees[sMethod]; + except KeyError: + raise RestDispException('Unsupported method %s' % (sMethod,), 405); + + # + # Walk the path till we find a handler for it. + # + iPath = 0; + while iPath < len(asPath): + try: + oTreeOrHandler = dTree[asPath[iPath]]; + except KeyError: + raise RestBadPathException('Path element #%u "%s" not found (path="%s")' % (iPath, asPath[iPath], self._sPath)); + iPath += 1; + if isinstance(oTreeOrHandler, dict): + dTree = oTreeOrHandler; + else: + # + # Call the handler. + # + self._iFirstHandlerPath = iPath; + self._iNextHandlerPath = iPath; + return oTreeOrHandler(); + + raise RestBadPathException('Empty path (%s)' % (self._sPath,)); + + def dispatchRequest(self): + """ + Dispatches the incoming request where the path is given as an argument. + + Will raise RestDispException on failure. + """ + + # + # Get the parameters. + # + try: + dParams = self._oSrvGlue.getParameters(); + except Exception as oXcpt: + raise RestDispException('Error retriving parameters: %s' % (oXcpt,), 500); + self._dParams = dParams; + + # + # Get the path parameter. + # + if self.ksParam_sPath not in dParams: + raise RestDispException('No "%s" parameter in request (params: %s)' % (self.ksParam_sPath, dParams,), 500); + self._sPath = self._getStringParam(self.ksParam_sPath); + assert utils.isString(self._sPath); + + return self._dispatchRequestCommon(); + diff --git a/src/VBox/ValidationKit/testmanager/core/schedgroup.py b/src/VBox/ValidationKit/testmanager/core/schedgroup.py new file mode 100755 index 00000000..a586a6bf --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/schedgroup.py @@ -0,0 +1,1352 @@ +# -*- coding: utf-8 -*- +# $Id: schedgroup.py $ + +""" +Test Manager - Scheduling Group. +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Standard python imports. +import unittest; + +# Validation Kit imports. +from testmanager.core.base import ModelDataBase, ModelDataBaseTestCase, ModelLogicBase, TMExceptionBase, \ + TMRowInUse, TMInvalidData, TMRowAlreadyExists, TMRowNotFound, \ + ChangeLogEntry, AttributeChangeEntry, AttributeChangeEntryPre; +from testmanager.core.buildsource import BuildSourceData; +from testmanager.core import db; +from testmanager.core.testcase import TestCaseData; +from testmanager.core.testcaseargs import TestCaseArgsData; +from testmanager.core.testbox import TestBoxLogic, TestBoxDataForSchedGroup; +from testmanager.core.testgroup import TestGroupData; +from testmanager.core.useraccount import UserAccountLogic; + + + +class SchedGroupMemberData(ModelDataBase): + """ + SchedGroupMember Data. + """ + + ksIdAttr = 'idSchedGroup'; + + ksParam_idSchedGroup = 'SchedGroupMember_idSchedGroup'; + ksParam_idTestGroup = 'SchedGroupMember_idTestGroup'; + ksParam_tsEffective = 'SchedGroupMember_tsEffective'; + ksParam_tsExpire = 'SchedGroupMember_tsExpire'; + ksParam_uidAuthor = 'SchedGroupMember_uidAuthor'; + ksParam_iSchedPriority = 'SchedGroupMember_iSchedPriority'; + ksParam_bmHourlySchedule = 'SchedGroupMember_bmHourlySchedule'; + ksParam_idTestGroupPreReq = 'SchedGroupMember_idTestGroupPreReq'; + + kasAllowNullAttributes = [ 'idSchedGroup', 'idTestGroup', 'tsEffective', 'tsExpire', + 'uidAuthor', 'bmHourlySchedule', 'idTestGroupPreReq' ]; + kiMin_iSchedPriority = 0; + kiMax_iSchedPriority = 32; + + kcDbColumns = 8 + + def __init__(self): + ModelDataBase.__init__(self); + + # + # Initialize with defaults. + # See the database for explanations of each of these fields. + # + self.idSchedGroup = None; + self.idTestGroup = None; + self.tsEffective = None; + self.tsExpire = None; + self.uidAuthor = None; + self.iSchedPriority = 16; + self.bmHourlySchedule = None; + self.idTestGroupPreReq = None; + + def initFromDbRow(self, aoRow): + """ + Re-initializes the data with a row from a SELECT * FROM SchedGroupMembers. + + Returns self. Raises exception if the row is None or otherwise invalid. + """ + + if aoRow is None: + raise TMRowNotFound('SchedGroupMember not found.'); + + self.idSchedGroup = aoRow[0]; + self.idTestGroup = aoRow[1]; + self.tsEffective = aoRow[2]; + self.tsExpire = aoRow[3]; + self.uidAuthor = aoRow[4]; + self.iSchedPriority = aoRow[5]; + self.bmHourlySchedule = aoRow[6]; ## @todo figure out how bitmaps are returned... + self.idTestGroupPreReq = aoRow[7]; + return self; + + +class SchedGroupMemberDataEx(SchedGroupMemberData): + """ + Extended SchedGroupMember data class. + This adds the testgroups. + """ + + def __init__(self): + SchedGroupMemberData.__init__(self); + self.oTestGroup = None; + + def initFromDbRow(self, aoRow): + """ + Re-initializes the data with a row from a query like this: + + SELECT SchedGroupMembers.*, TestGroups.* + FROM SchedGroupMembers + JOIN TestGroups + ON (SchedGroupMembers.idTestGroup = TestGroups.idTestGroup); + + Returns self. Raises exception if the row is None or otherwise invalid. + """ + SchedGroupMemberData.initFromDbRow(self, aoRow); + self.oTestGroup = TestGroupData().initFromDbRow(aoRow[SchedGroupMemberData.kcDbColumns:]); + return self; + + def getDataAttributes(self): + asAttributes = SchedGroupMemberData.getDataAttributes(self); + asAttributes.remove('oTestGroup'); + return asAttributes; + + def _validateAndConvertWorker(self, asAllowNullAttributes, oDb, enmValidateFor = ModelDataBase.ksValidateFor_Other): + dErrors = SchedGroupMemberData._validateAndConvertWorker(self, asAllowNullAttributes, oDb, enmValidateFor); + if self.ksParam_idTestGroup not in dErrors: + self.oTestGroup = TestGroupData(); + try: + self.oTestGroup.initFromDbWithId(oDb, self.idTestGroup); + except Exception as oXcpt: + self.oTestGroup = TestGroupData() + dErrors[self.ksParam_idTestGroup] = str(oXcpt); + return dErrors; + + + + +class SchedGroupData(ModelDataBase): + """ + SchedGroup Data. + """ + + ## @name TestBoxState_T + # @{ + ksScheduler_BestEffortContinuousIntegration = 'bestEffortContinousItegration'; # sic*2 + ksScheduler_Reserved = 'reserved'; + ## @} + + + ksIdAttr = 'idSchedGroup'; + + ksParam_idSchedGroup = 'SchedGroup_idSchedGroup'; + ksParam_tsEffective = 'SchedGroup_tsEffective'; + ksParam_tsExpire = 'SchedGroup_tsExpire'; + ksParam_uidAuthor = 'SchedGroup_uidAuthor'; + ksParam_sName = 'SchedGroup_sName'; + ksParam_sDescription = 'SchedGroup_sDescription'; + ksParam_fEnabled = 'SchedGroup_fEnabled'; + ksParam_enmScheduler = 'SchedGroup_enmScheduler'; + ksParam_idBuildSrc = 'SchedGroup_idBuildSrc'; + ksParam_idBuildSrcTestSuite = 'SchedGroup_idBuildSrcTestSuite'; + ksParam_sComment = 'SchedGroup_sComment'; + + kasAllowNullAttributes = ['idSchedGroup', 'tsEffective', 'tsExpire', 'uidAuthor', 'sDescription', + 'idBuildSrc', 'idBuildSrcTestSuite', 'sComment' ]; + kasValidValues_enmScheduler = [ ksScheduler_BestEffortContinuousIntegration, ]; + + kcDbColumns = 11; + + # Scheduler types + kasSchedulerDesc = \ + [ + ( ksScheduler_BestEffortContinuousIntegration, 'Best-Effort-Continuous-Integration (BECI) scheduler.', ''), + ] + + def __init__(self): + ModelDataBase.__init__(self); + + # + # Initialize with defaults. + # See the database for explanations of each of these fields. + # + self.idSchedGroup = None; + self.tsEffective = None; + self.tsExpire = None; + self.uidAuthor = None; + self.sName = None; + self.sDescription = None; + self.fEnabled = None; + self.enmScheduler = SchedGroupData.ksScheduler_BestEffortContinuousIntegration; + self.idBuildSrc = None; + self.idBuildSrcTestSuite = None; + self.sComment = None; + + def initFromDbRow(self, aoRow): + """ + Re-initializes the data with a row from a SELECT * FROM SchedGroups. + + Returns self. Raises exception if the row is None or otherwise invalid. + """ + + if aoRow is None: + raise TMRowNotFound('SchedGroup not found.'); + + self.idSchedGroup = aoRow[0]; + self.tsEffective = aoRow[1]; + self.tsExpire = aoRow[2]; + self.uidAuthor = aoRow[3]; + self.sName = aoRow[4]; + self.sDescription = aoRow[5]; + self.fEnabled = aoRow[6]; + self.enmScheduler = aoRow[7]; + self.idBuildSrc = aoRow[8]; + self.idBuildSrcTestSuite = aoRow[9]; + self.sComment = aoRow[10]; + return self; + + def initFromDbWithId(self, oDb, idSchedGroup, tsNow = None, sPeriodBack = None): + """ + Initialize the object from the database. + """ + oDb.execute(self.formatSimpleNowAndPeriodQuery(oDb, + 'SELECT *\n' + 'FROM SchedGroups\n' + 'WHERE idSchedGroup = %s\n' + , ( idSchedGroup,), tsNow, sPeriodBack)); + aoRow = oDb.fetchOne() + if aoRow is None: + raise TMRowNotFound('idSchedGroup=%s not found (tsNow=%s, sPeriodBack=%s)' % (idSchedGroup, tsNow, sPeriodBack)); + return self.initFromDbRow(aoRow); + + +class SchedGroupDataEx(SchedGroupData): + """ + Extended scheduling group data. + + Note! Similar to TestGroupDataEx. + """ + + ksParam_aoMembers = 'SchedGroup_aoMembers'; + ksParam_aoTestBoxes = 'SchedGroup_aoTestboxes'; + kasAltArrayNull = [ 'aoMembers', 'aoTestboxes' ]; + + ## Helper parameter containing the comma separated list with the IDs of + # potential members found in the parameters. + ksParam_aidTestGroups = 'TestGroupDataEx_aidTestGroups'; + ## Ditto for testbox meembers. + ksParam_aidTestBoxes = 'TestGroupDataEx_aidTestBoxes'; + + + def __init__(self): + SchedGroupData.__init__(self); + self.aoMembers = [] # type: list[SchedGroupMemberDataEx] + self.aoTestBoxes = [] # type: list[TestBoxDataForSchedGroup] + + # The two build sources for the sake of convenience. + self.oBuildSrc = None # type: BuildSourceData + self.oBuildSrcValidationKit = None # type: BuildSourceData + + def _initExtraMembersFromDb(self, oDb, tsNow = None, sPeriodBack = None): + """ + Worker shared by the initFromDb* methods. + Returns self. Raises exception if no row or database error. + """ + # + # Clear all members upfront so the object has some kind of consistency + # if anything below raises exceptions. + # + self.oBuildSrc = None; + self.oBuildSrcValidationKit = None; + self.aoTestBoxes = []; + self.aoMembers = []; + + # + # Build source. + # + if self.idBuildSrc: + self.oBuildSrc = BuildSourceData().initFromDbWithId(oDb, self.idBuildSrc, tsNow, sPeriodBack); + + if self.idBuildSrcTestSuite: + self.oBuildSrcValidationKit = BuildSourceData().initFromDbWithId(oDb, self.idBuildSrcTestSuite, + tsNow, sPeriodBack); + + # + # Test Boxes. + # + self.aoTestBoxes = TestBoxLogic(oDb).fetchForSchedGroup(self.idSchedGroup, tsNow); + + # + # Test groups. + # The fetchForChangeLog method makes ASSUMPTIONS about sorting! + # + oDb.execute('SELECT SchedGroupMembers.*, TestGroups.*\n' + 'FROM SchedGroupMembers\n' + 'LEFT OUTER JOIN TestGroups ON (SchedGroupMembers.idTestGroup = TestGroups.idTestGroup)\n' + 'WHERE SchedGroupMembers.idSchedGroup = %s\n' + + self.formatSimpleNowAndPeriod(oDb, tsNow, sPeriodBack, sTablePrefix = 'SchedGroupMembers.') + + self.formatSimpleNowAndPeriod(oDb, tsNow, sPeriodBack, sTablePrefix = 'TestGroups.') + + 'ORDER BY SchedGroupMembers.idTestGroupPreReq ASC NULLS FIRST,\n' + ' TestGroups.sName,\n' + ' SchedGroupMembers.idTestGroup\n' + , (self.idSchedGroup,)); + for aoRow in oDb.fetchAll(): + self.aoMembers.append(SchedGroupMemberDataEx().initFromDbRow(aoRow)); + return self; + + def initFromDbRowEx(self, aoRow, oDb, tsNow = None): + """ + Reinitialize from a SELECT * FROM SchedGroups row. Will query the + necessary additional data from oDb using tsNow. + Returns self. Raises exception if no row or database error. + """ + SchedGroupData.initFromDbRow(self, aoRow); + return self._initExtraMembersFromDb(oDb, tsNow); + + def initFromDbWithId(self, oDb, idSchedGroup, tsNow = None, sPeriodBack = None): + """ + Initialize the object from the database. + """ + SchedGroupData.initFromDbWithId(self, oDb, idSchedGroup, tsNow, sPeriodBack); + return self._initExtraMembersFromDb(oDb, tsNow, sPeriodBack); + + def getDataAttributes(self): + asAttributes = SchedGroupData.getDataAttributes(self); + asAttributes.remove('oBuildSrc'); + asAttributes.remove('oBuildSrcValidationKit'); + return asAttributes; + + def getAttributeParamNullValues(self, sAttr): + if sAttr not in [ 'aoMembers', 'aoTestBoxes' ]: + return SchedGroupData.getAttributeParamNullValues(self, sAttr); + return ['', [], None]; + + def convertParamToAttribute(self, sAttr, sParam, oValue, oDisp, fStrict): + aoNewValue = []; + if sAttr == 'aoMembers': + aidSelected = oDisp.getListOfIntParams(sParam, iMin = 1, iMax = 0x7ffffffe, aiDefaults = []) + sIds = oDisp.getStringParam(self.ksParam_aidTestGroups, sDefault = ''); + for idTestGroup in sIds.split(','): + try: idTestGroup = int(idTestGroup); + except: pass; + oDispWrapper = self.DispWrapper(oDisp, '%s[%s][%%s]' % (SchedGroupDataEx.ksParam_aoMembers, idTestGroup,)) + oMember = SchedGroupMemberDataEx().initFromParams(oDispWrapper, fStrict = False); + if idTestGroup in aidSelected: + oMember.idTestGroup = idTestGroup; + aoNewValue.append(oMember); + elif sAttr == 'aoTestBoxes': + aidSelected = oDisp.getListOfIntParams(sParam, iMin = 1, iMax = 0x7ffffffe, aiDefaults = []) + sIds = oDisp.getStringParam(self.ksParam_aidTestBoxes, sDefault = ''); + for idTestBox in sIds.split(','): + try: idTestBox = int(idTestBox); + except: pass; + oDispWrapper = self.DispWrapper(oDisp, '%s[%s][%%s]' % (SchedGroupDataEx.ksParam_aoTestBoxes, idTestBox,)) + oBoxInGrp = TestBoxDataForSchedGroup().initFromParams(oDispWrapper, fStrict = False); + if idTestBox in aidSelected: + oBoxInGrp.idTestBox = idTestBox; + aoNewValue.append(oBoxInGrp); + else: + return SchedGroupData.convertParamToAttribute(self, sAttr, sParam, oValue, oDisp, fStrict); + return aoNewValue; + + def _validateAndConvertAttribute(self, sAttr, sParam, oValue, aoNilValues, fAllowNull, oDb): + if sAttr not in [ 'aoMembers', 'aoTestBoxes' ]: + return SchedGroupData._validateAndConvertAttribute(self, sAttr, sParam, oValue, aoNilValues, fAllowNull, oDb); + + if oValue in aoNilValues: + return ([], None); + + asErrors = []; + aoNewMembers = []; + if sAttr == 'aoMembers': + asAllowNulls = ['bmHourlySchedule', 'idTestGroupPreReq', 'tsEffective', 'tsExpire', 'uidAuthor', ]; + if self.idSchedGroup in [None, '-1', -1]: + asAllowNulls.append('idSchedGroup'); # Probably new group, so allow null scheduling group. + + for oOldMember in oValue: + oNewMember = SchedGroupMemberDataEx().initFromOther(oOldMember); + aoNewMembers.append(oNewMember); + + dErrors = oNewMember.validateAndConvertEx(asAllowNulls, oDb, ModelDataBase.ksValidateFor_Other); + if dErrors: + asErrors.append(str(dErrors)); + + if not asErrors: + for i, _ in enumerate(aoNewMembers): + idTestGroup = aoNewMembers[i]; + for j in range(i + 1, len(aoNewMembers)): + if aoNewMembers[j].idTestGroup == idTestGroup: + asErrors.append('Duplicate test group #%d!' % (idTestGroup, )); + break; + else: + asAllowNulls = list(TestBoxDataForSchedGroup.kasAllowNullAttributes); + if self.idSchedGroup in [None, '-1', -1]: + asAllowNulls.append('idSchedGroup'); # Probably new group, so allow null scheduling group. + + for oOldMember in oValue: + oNewMember = TestBoxDataForSchedGroup().initFromOther(oOldMember); + aoNewMembers.append(oNewMember); + + dErrors = oNewMember.validateAndConvertEx(asAllowNulls, oDb, ModelDataBase.ksValidateFor_Other); + if dErrors: + asErrors.append(str(dErrors)); + + if not asErrors: + for i, _ in enumerate(aoNewMembers): + idTestBox = aoNewMembers[i]; + for j in range(i + 1, len(aoNewMembers)): + if aoNewMembers[j].idTestBox == idTestBox: + asErrors.append('Duplicate test box #%d!' % (idTestBox, )); + break; + + return (aoNewMembers, None if not asErrors else '<br>\n'.join(asErrors)); + + def _validateAndConvertWorker(self, asAllowNullAttributes, oDb, enmValidateFor = ModelDataBase.ksValidateFor_Other): + dErrors = SchedGroupData._validateAndConvertWorker(self, asAllowNullAttributes, oDb, enmValidateFor); + + # + # Fetch the extended build source bits. + # + if self.ksParam_idBuildSrc not in dErrors: + if self.idBuildSrc in self.getAttributeParamNullValues('idBuildSrc') \ + or self.idBuildSrc is None: + self.oBuildSrc = None; + else: + try: + self.oBuildSrc = BuildSourceData().initFromDbWithId(oDb, self.idBuildSrc); + except Exception as oXcpt: + self.oBuildSrc = BuildSourceData(); + dErrors[self.ksParam_idBuildSrc] = str(oXcpt); + + if self.ksParam_idBuildSrcTestSuite not in dErrors: + if self.idBuildSrcTestSuite in self.getAttributeParamNullValues('idBuildSrcTestSuite') \ + or self.idBuildSrcTestSuite is None: + self.oBuildSrcValidationKit = None; + else: + try: + self.oBuildSrcValidationKit = BuildSourceData().initFromDbWithId(oDb, self.idBuildSrcTestSuite); + except Exception as oXcpt: + self.oBuildSrcValidationKit = BuildSourceData(); + dErrors[self.ksParam_idBuildSrcTestSuite] = str(oXcpt); + + return dErrors; + + + +class SchedGroupLogic(ModelLogicBase): # pylint: disable=too-few-public-methods + """ + SchedGroup logic. + """ + + def __init__(self, oDb): + ModelLogicBase.__init__(self, oDb); + self.dCache = None; + + # + # Standard methods. + # + + def fetchForListing(self, iStart, cMaxRows, tsNow, aiSortColumns = None): + """ + Fetches build sources. + + Returns an array (list) of BuildSourceData items, empty list if none. + Raises exception on error. + """ + _ = aiSortColumns; + + if tsNow is None: + self._oDb.execute('SELECT *\n' + 'FROM SchedGroups\n' + 'WHERE tsExpire = \'infinity\'::TIMESTAMP\n' + 'ORDER BY fEnabled DESC, sName DESC\n' + 'LIMIT %s OFFSET %s\n' + , (cMaxRows, iStart,)); + else: + self._oDb.execute('SELECT *\n' + 'FROM SchedGroups\n' + 'WHERE tsExpire > %s\n' + ' AND tsEffective <= %s\n' + 'ORDER BY fEnabled DESC, sName DESC\n' + 'LIMIT %s OFFSET %s\n' + , (tsNow, tsNow, cMaxRows, iStart,)); + + aoRet = []; + for aoRow in self._oDb.fetchAll(): + aoRet.append(SchedGroupDataEx().initFromDbRowEx(aoRow, self._oDb, tsNow)); + return aoRet; + + def fetchForChangeLog(self, idSchedGroup, iStart, cMaxRows, tsNow): # pylint: disable=too-many-locals,too-many-statements + """ + Fetches change log entries for a scheduling group. + + Returns an array of ChangeLogEntry instance and an indicator whether + there are more entries. + Raises exception on error. + """ + + ## @todo calc changes to scheduler group! + + if tsNow is None: + tsNow = self._oDb.getCurrentTimestamp(); + + # + # First gather the change log timeline using the effective dates. + # (ASSUMES that we'll always have a separate delete entry, rather + # than just setting tsExpire.) + # + self._oDb.execute(''' +( +SELECT tsEffective, + uidAuthor +FROM SchedGroups +WHERE idSchedGroup = %s + AND tsEffective <= %s +ORDER BY tsEffective DESC +) UNION ( +SELECT CASE WHEN tsEffective + %s::INTERVAL = tsExpire THEN tsExpire ELSE tsEffective END, + uidAuthor +FROM SchedGroupMembers +WHERE idSchedGroup = %s + AND tsEffective <= %s +ORDER BY tsEffective DESC +) UNION ( +SELECT CASE WHEN tsEffective + %s::INTERVAL = tsExpire THEN tsExpire ELSE tsEffective END, + uidAuthor +FROM TestBoxesInSchedGroups +WHERE idSchedGroup = %s + AND tsEffective <= %s +ORDER BY tsEffective DESC +) +ORDER BY tsEffective DESC +LIMIT %s OFFSET %s +''', (idSchedGroup, tsNow, + db.dbOneTickIntervalString(), idSchedGroup, tsNow, + db.dbOneTickIntervalString(), idSchedGroup, tsNow, + cMaxRows + 1, iStart, )); + + aoEntries = [] # type: list[ChangeLogEntry] + tsPrevious = tsNow; + for aoDbRow in self._oDb.fetchAll(): + (tsEffective, uidAuthor) = aoDbRow; + aoEntries.append(ChangeLogEntry(uidAuthor, None, tsEffective, tsPrevious, None, None, [])); + tsPrevious = db.dbTimestampPlusOneTick(tsEffective); + + if True: # pylint: disable=using-constant-test + # + # Fetch data for each for each change log entry point. + # + # We add one tick to the timestamp here to skip past delete records + # that only there to record the user doing the deletion. + # + for iEntry, oEntry in enumerate(aoEntries): + oEntry.oNewRaw = SchedGroupDataEx().initFromDbWithId(self._oDb, idSchedGroup, oEntry.tsEffective); + if iEntry > 0: + aoEntries[iEntry - 1].oOldRaw = oEntry.oNewRaw; + + # Chop off the +1 entry, if any. + fMore = len(aoEntries) > cMaxRows; + if fMore: + aoEntries = aoEntries[:-1]; + + # Figure out the changes. + for oEntry in aoEntries: + oOld = oEntry.oOldRaw; + if not oOld: + break; + oNew = oEntry.oNewRaw; + aoChanges = oEntry.aoChanges; + for sAttr in oNew.getDataAttributes(): + if sAttr in [ 'tsEffective', 'tsExpire', 'uidAuthor', ]: + continue; + oOldAttr = getattr(oOld, sAttr); + oNewAttr = getattr(oNew, sAttr); + if oOldAttr == oNewAttr: + continue; + if sAttr in [ 'aoMembers', 'aoTestBoxes', ]: + iNew = 0; + iOld = 0; + asNewAttr = []; + asOldAttr = []; + if sAttr == 'aoMembers': + # ASSUMES aoMembers is sorted by idTestGroupPreReq (nulls first), oTestGroup.sName, idTestGroup! + while iNew < len(oNewAttr) and iOld < len(oOldAttr): + if oNewAttr[iNew].idTestGroup == oOldAttr[iOld].idTestGroup: + if oNewAttr[iNew].idTestGroupPreReq != oOldAttr[iOld].idTestGroupPreReq: + if oNewAttr[iNew].idTestGroupPreReq is None: + asOldAttr.append('Dropped test group #%s (%s) dependency on #%s' + % (oNewAttr[iNew].idTestGroup, oNewAttr[iNew].oTestGroup.sName, + oOldAttr[iOld].idTestGroupPreReq)); + elif oOldAttr[iOld].idTestGroupPreReq is None: + asNewAttr.append('Added test group #%s (%s) dependency on #%s' + % (oNewAttr[iNew].idTestGroup, oNewAttr[iNew].oTestGroup.sName, + oNewAttr[iOld].idTestGroupPreReq)); + else: + asNewAttr.append('Test group #%s (%s) dependency on #%s' + % (oNewAttr[iNew].idTestGroup, oNewAttr[iNew].oTestGroup.sName, + oNewAttr[iNew].idTestGroupPreReq)); + asOldAttr.append('Test group #%s (%s) dependency on #%s' + % (oNewAttr[iNew].idTestGroup, oNewAttr[iNew].oTestGroup.sName, + oOldAttr[iOld].idTestGroupPreReq)); + if oNewAttr[iNew].iSchedPriority != oOldAttr[iOld].iSchedPriority: + asNewAttr.append('Test group #%s (%s) priority %s' + % (oNewAttr[iNew].idTestGroup, oNewAttr[iNew].oTestGroup.sName, + oNewAttr[iNew].iSchedPriority)); + asOldAttr.append('Test group #%s (%s) priority %s' + % (oNewAttr[iNew].idTestGroup, oNewAttr[iNew].oTestGroup.sName, + oOldAttr[iOld].iSchedPriority)); + iNew += 1; + iOld += 1; + elif oNewAttr[iNew].oTestGroup.sName < oOldAttr[iOld].oTestGroup.sName \ + or ( oNewAttr[iNew].oTestGroup.sName == oOldAttr[iOld].oTestGroup.sName + and oNewAttr[iNew].idTestGroup < oOldAttr[iOld].idTestGroup): + asNewAttr.append('New test group #%s - %s' + % (oNewAttr[iNew].idTestGroup, oNewAttr[iNew].oTestGroup.sName)); + iNew += 1; + else: + asOldAttr.append('Removed test group #%s - %s' + % (oOldAttr[iOld].idTestGroup, oOldAttr[iOld].oTestGroup.sName)); + iOld += 1; + while iNew < len(oNewAttr): + asNewAttr.append('New test group #%s - %s' + % (oNewAttr[iNew].idTestGroup, oNewAttr[iNew].oTestGroup.sName)); + iNew += 1; + while iOld < len(oOldAttr): + asOldAttr.append('Removed test group #%s - %s' + % (oOldAttr[iOld].idTestGroup, oOldAttr[iOld].oTestGroup.sName)); + iOld += 1; + else: + dNewIds = { oBoxInGrp.idTestBox: oBoxInGrp for oBoxInGrp in oNewAttr }; + dOldIds = { oBoxInGrp.idTestBox: oBoxInGrp for oBoxInGrp in oOldAttr }; + hCommonIds = set(dNewIds.keys()) & set(dOldIds.keys()); + for idTestBox in hCommonIds: + oNewBoxInGrp = dNewIds[idTestBox]; + oOldBoxInGrp = dOldIds[idTestBox]; + if oNewBoxInGrp.iSchedPriority != oOldBoxInGrp.iSchedPriority: + asNewAttr.append('Test box \'%s\' (#%s) priority %s' + % (getattr(oNewBoxInGrp.oTestBox, 'sName', '[Partial DB]'), + oNewBoxInGrp.idTestBox, oNewBoxInGrp.iSchedPriority)); + asOldAttr.append('Test box \'%s\' (#%s) priority %s' + % (getattr(oOldBoxInGrp.oTestBox, 'sName', '[Partial DB]'), + oOldBoxInGrp.idTestBox, oOldBoxInGrp.iSchedPriority)); + asNewAttr = sorted(asNewAttr); + asOldAttr = sorted(asOldAttr); + for idTestBox in set(dNewIds.keys()) - hCommonIds: + oNewBoxInGrp = dNewIds[idTestBox]; + asNewAttr.append('New test box \'%s\' (#%s) priority %s' + % (getattr(oNewBoxInGrp.oTestBox, 'sName', '[Partial DB]'), + oNewBoxInGrp.idTestBox, oNewBoxInGrp.iSchedPriority)); + for idTestBox in set(dOldIds.keys()) - hCommonIds: + oOldBoxInGrp = dOldIds[idTestBox]; + asOldAttr.append('Removed test box \'%s\' (#%s) priority %s' + % (getattr(oOldBoxInGrp.oTestBox, 'sName', '[Partial DB]'), + oOldBoxInGrp.idTestBox, oOldBoxInGrp.iSchedPriority)); + + if asNewAttr or asOldAttr: + aoChanges.append(AttributeChangeEntryPre(sAttr, oNewAttr, oOldAttr, + '\n'.join(asNewAttr), '\n'.join(asOldAttr))); + else: + aoChanges.append(AttributeChangeEntry(sAttr, oNewAttr, oOldAttr, str(oNewAttr), str(oOldAttr))); + + else: + ## + ## @todo Incomplete: A more complicate apporach, probably faster though. + ## + def findEntry(tsEffective, iPrev = 0): + """ Find entry with matching effective + expiration time """ + self._oDb.dprint('findEntry: iPrev=%s len(aoEntries)=%s tsEffective=%s' % (iPrev, len(aoEntries), tsEffective)); + while iPrev < len(aoEntries): + self._oDb.dprint('%s iPrev=%u' % (aoEntries[iPrev].tsEffective, iPrev, )); + if aoEntries[iPrev].tsEffective > tsEffective: + iPrev += 1; + elif aoEntries[iPrev].tsEffective == tsEffective: + self._oDb.dprint('hit %u' % (iPrev,)); + return iPrev; + else: + break; + self._oDb.dprint('%s not found!' % (tsEffective,)); + return -1; + + fMore = True; + + # + # Track scheduling group changes. Not terribly efficient for large cMaxRows + # values, but not in the mood for figure out if there is any way to optimize that. + # + self._oDb.execute(''' +SELECT * +FROM SchedGroups +WHERE idSchedGroup = %s + AND tsEffective <= %s +ORDER BY tsEffective DESC +LIMIT %s''', (idSchedGroup, aoEntries[0].tsEffective, cMaxRows + 1,)); + + iEntry = 0; + aaoRows = self._oDb.fetchAll(); + for iRow, oRow in enumerate(aaoRows): + oNew = SchedGroupDataEx().initFromDbRow(oRow); + iEntry = findEntry(oNew.tsEffective, iEntry); + self._oDb.dprint('iRow=%s iEntry=%s' % (iRow, iEntry)); + if iEntry < 0: + break; + oEntry = aoEntries[iEntry]; + aoChanges = oEntry.aoChanges; + oEntry.oNewRaw = oNew; + if iRow + 1 < len(aaoRows): + oOld = SchedGroupDataEx().initFromDbRow(aaoRows[iRow + 1]); + self._oDb.dprint('oOld=%s' % (oOld,)); + for sAttr in oNew.getDataAttributes(): + if sAttr not in [ 'tsEffective', 'tsExpire', 'uidAuthor', ]: + oOldAttr = getattr(oOld, sAttr); + oNewAttr = getattr(oNew, sAttr); + if oOldAttr != oNewAttr: + aoChanges.append(AttributeChangeEntry(sAttr, oNewAttr, oOldAttr, str(oNewAttr), str(oOldAttr))); + else: + self._oDb.dprint('New'); + + # + # ... + # + + # FInally + UserAccountLogic(self._oDb).resolveChangeLogAuthors(aoEntries); + return (aoEntries, fMore); + + + def addEntry(self, oData, uidAuthor, fCommit = False): + """Add Scheduling Group record""" + + # + # Validate. + # + dDataErrors = oData.validateAndConvert(self._oDb, oData.ksValidateFor_Add); + if dDataErrors: + raise TMInvalidData('Invalid data passed to addEntry: %s' % (dDataErrors,)); + if self.exists(oData.sName): + raise TMRowAlreadyExists('Scheduling group "%s" already exists.' % (oData.sName,)); + + # + # Add it. + # + self._oDb.execute('INSERT INTO SchedGroups (\n' + ' uidAuthor,\n' + ' sName,\n' + ' sDescription,\n' + ' fEnabled,\n' + ' enmScheduler,\n' + ' idBuildSrc,\n' + ' idBuildSrcTestSuite,\n' + ' sComment)\n' + 'VALUES (%s, %s, %s, %s, %s, %s, %s, %s)\n' + 'RETURNING idSchedGroup\n' + , ( uidAuthor, + oData.sName, + oData.sDescription, + oData.fEnabled, + oData.enmScheduler, + oData.idBuildSrc, + oData.idBuildSrcTestSuite, + oData.sComment )); + idSchedGroup = self._oDb.fetchOne()[0]; + oData.idSchedGroup = idSchedGroup; + + for oBoxInGrp in oData.aoTestBoxes: + oBoxInGrp.idSchedGroup = idSchedGroup; + self._addSchedGroupTestBox(uidAuthor, oBoxInGrp); + + for oMember in oData.aoMembers: + oMember.idSchedGroup = idSchedGroup; + self._addSchedGroupMember(uidAuthor, oMember); + + self._oDb.maybeCommit(fCommit); + return True; + + def editEntry(self, oData, uidAuthor, fCommit = False): + """Edit Scheduling Group record""" + + # + # Validate input and retrieve the old data. + # + dErrors = oData.validateAndConvert(self._oDb, oData.ksValidateFor_Edit); + if dErrors: + raise TMInvalidData('editEntry got invalid data: %s' % (dErrors,)); + self._assertUnique(oData.sName, oData.idSchedGroup); + oOldData = SchedGroupDataEx().initFromDbWithId(self._oDb, oData.idSchedGroup); + + # + # Make the changes. + # + if not oData.isEqualEx(oOldData, [ 'tsEffective', 'tsExpire', 'uidAuthor', 'aoMembers', 'aoTestBoxes', + 'oBuildSrc', 'oBuildSrcValidationKit', ]): + self._historizeEntry(oData.idSchedGroup); + self._readdEntry(uidAuthor, oData); + + # Remove groups. + for oOld in oOldData.aoMembers: + fRemove = True; + for oNew in oData.aoMembers: + if oNew.idTestGroup == oOld.idTestGroup: + fRemove = False; + break; + if fRemove: + self._removeSchedGroupMember(uidAuthor, oOld); + + # Add / modify groups. + for oMember in oData.aoMembers: + oOldMember = None; + for oOld in oOldData.aoMembers: + if oOld.idTestGroup == oMember.idTestGroup: + oOldMember = oOld; + break; + + oMember.idSchedGroup = oData.idSchedGroup; + if oOldMember is None: + self._addSchedGroupMember(uidAuthor, oMember); + elif not oMember.isEqualEx(oOldMember, ['tsEffective', 'tsExpire', 'uidAuthor', 'oTestGroup']): + self._historizeSchedGroupMember(oMember); + self._addSchedGroupMember(uidAuthor, oMember); + + # Remove testboxes. + for oOld in oOldData.aoTestBoxes: + fRemove = True; + for oNew in oData.aoTestBoxes: + if oNew.idTestBox == oOld.idTestBox: + fRemove = False; + break; + if fRemove: + self._removeSchedGroupTestBox(uidAuthor, oOld); + + # Add / modify testboxes. + for oBoxInGrp in oData.aoTestBoxes: + oOldBoxInGrp = None; + for oOld in oOldData.aoTestBoxes: + if oOld.idTestBox == oBoxInGrp.idTestBox: + oOldBoxInGrp = oOld; + break; + + oBoxInGrp.idSchedGroup = oData.idSchedGroup; + if oOldBoxInGrp is None: + self._addSchedGroupTestBox(uidAuthor, oBoxInGrp); + elif not oBoxInGrp.isEqualEx(oOldBoxInGrp, ['tsEffective', 'tsExpire', 'uidAuthor', 'oTestBox']): + self._historizeSchedGroupTestBox(oBoxInGrp); + self._addSchedGroupTestBox(uidAuthor, oBoxInGrp); + + self._oDb.maybeCommit(fCommit); + return True; + + def removeEntry(self, uidAuthor, idSchedGroup, fCascade = False, fCommit = False): + """ + Deletes a scheduling group. + """ + _ = fCascade; + + # + # Input validation and retrival of current data. + # + if idSchedGroup == 1: + raise TMRowInUse('Cannot remove the default scheduling group (id 1).'); + oData = SchedGroupDataEx().initFromDbWithId(self._oDb, idSchedGroup); + + # + # Remove the test box member records. + # + for oBoxInGrp in oData.aoTestBoxes: + self._removeSchedGroupTestBox(uidAuthor, oBoxInGrp); + self._oDb.execute('UPDATE TestBoxesInSchedGroups\n' + 'SET tsExpire = CURRENT_TIMESTAMP\n' + 'WHERE idSchedGroup = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , (idSchedGroup,)); + + # + # Remove the test group member records. + # + for oMember in oData.aoMembers: + self._removeSchedGroupMember(uidAuthor, oMember); + self._oDb.execute('UPDATE SchedGroupMembers\n' + 'SET tsExpire = CURRENT_TIMESTAMP\n' + 'WHERE idSchedGroup = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , (idSchedGroup,)); + + # + # Now the SchedGroups entry. + # + (tsCur, tsCurMinusOne) = self._oDb.getCurrentTimestamps(); + if oData.tsEffective not in (tsCur, tsCurMinusOne): + self._historizeEntry(idSchedGroup, tsCurMinusOne); + self._readdEntry(uidAuthor, oData, tsCurMinusOne); + self._historizeEntry(idSchedGroup); + self._oDb.execute('UPDATE SchedGroups\n' + 'SET tsExpire = CURRENT_TIMESTAMP\n' + 'WHERE idSchedGroup = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , (idSchedGroup,)) + + self._oDb.maybeCommit(fCommit) + return True; + + + def cachedLookup(self, idSchedGroup): + """ + Looks up the most recent SchedGroupData object for idSchedGroup + via an object cache. + + Returns a shared SchedGroupData object. None if not found. + Raises exception on DB error. + """ + if self.dCache is None: + self.dCache = self._oDb.getCache('SchedGroup'); + + oEntry = self.dCache.get(idSchedGroup, None); + if oEntry is None: + self._oDb.execute('SELECT *\n' + 'FROM SchedGroups\n' + 'WHERE idSchedGroup = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , (idSchedGroup, )); + if self._oDb.getRowCount() == 0: + # Maybe it was deleted, try get the last entry. + self._oDb.execute('SELECT *\n' + 'FROM SchedGroups\n' + 'WHERE idSchedGroup = %s\n' + 'ORDER BY tsExpire DESC\n' + 'LIMIT 1\n' + , (idSchedGroup, )); + elif self._oDb.getRowCount() > 1: + raise self._oDb.integrityException('%s infinity rows for %s' % (self._oDb.getRowCount(), idSchedGroup)); + + if self._oDb.getRowCount() == 1: + oEntry = SchedGroupData().initFromDbRow(self._oDb.fetchOne()); + self.dCache[idSchedGroup] = oEntry; + return oEntry; + + + # + # Other methods. + # + + def fetchOrderedByName(self, tsNow = None): + """ + Return list of objects of type SchedGroups ordered by name. + May raise exception on database error. + """ + if tsNow is None: + self._oDb.execute('SELECT *\n' + 'FROM SchedGroups\n' + 'WHERE tsExpire = \'infinity\'::TIMESTAMP\n' + 'ORDER BY sName ASC\n'); + else: + self._oDb.execute('SELECT *\n' + 'FROM SchedGroups\n' + 'WHERE tsExpire > %s\n' + ' AND tsEffective <= %s\n' + 'ORDER BY sName ASC\n' + , (tsNow, tsNow,)); + aoRet = [] + for _ in range(self._oDb.getRowCount()): + aoRet.append(SchedGroupData().initFromDbRow(self._oDb.fetchOne())); + return aoRet; + + + def getAll(self, tsEffective = None): + """ + Gets the list of all scheduling groups. + Returns an array of SchedGroupData instances. + """ + if tsEffective is None: + self._oDb.execute('SELECT *\n' + 'FROM SchedGroups\n' + 'WHERE tsExpire = \'infinity\'::TIMESTAMP\n'); + else: + self._oDb.execute('SELECT *\n' + 'FROM SchedGroups\n' + 'WHERE tsExpire > %s\n' + ' AND tsEffective <= %s\n' + , (tsEffective, tsEffective)); + aoRet = []; + for aoRow in self._oDb.fetchAll(): + aoRet.append(SchedGroupData().initFromDbRow(aoRow)); + return aoRet; + + def getSchedGroupsForCombo(self, tsEffective = None): + """ + Gets the list of active scheduling groups for a combo box. + Returns an array of (value [idSchedGroup], drop-down-name [sName], + hover-text [sDescription]) tuples. + """ + if tsEffective is None: + self._oDb.execute('SELECT idSchedGroup, sName, sDescription\n' + 'FROM SchedGroups\n' + 'WHERE tsExpire = \'infinity\'::TIMESTAMP\n' + 'ORDER BY sName'); + else: + self._oDb.execute('SELECT idSchedGroup, sName, sDescription\n' + 'FROM SchedGroups\n' + 'WHERE tsExpire > %s\n' + ' AND tsEffective <= %s\n' + 'ORDER BY sName' + , (tsEffective, tsEffective)); + return self._oDb.fetchAll(); + + + def getMembers(self, idSchedGroup, tsEffective = None): + """ + Gets the scheduling groups members for the given scheduling group. + + Returns an array of SchedGroupMemberDataEx instances (sorted by + priority (descending) and idTestGroup). May raise exception DB error. + """ + + if tsEffective is None: + self._oDb.execute('SELECT *\n' + 'FROM SchedGroupMembers, TestGroups\n' + 'WHERE SchedGroupMembers.idSchedGroup = %s\n' + ' AND SchedGroupMembers.tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND TestGroups.idTestGroup = SchedGroupMembers.idTestGroup\n' + ' AND TestGroups.tsExpire = \'infinity\'::TIMESTAMP\n' + 'ORDER BY SchedGroupMembers.iSchedPriority DESC, SchedGroupMembers.idTestGroup\n' + , (idSchedGroup,)); + else: + self._oDb.execute('SELECT *\n' + 'FROM SchedGroupMembers, TestGroups\n' + 'WHERE SchedGroupMembers.idSchedGroup = %s\n' + ' AND SchedGroupMembers.tsExpire < %s\n' + ' AND SchedGroupMembers.tsEffective >= %s\n' + ' AND TestGroups.idTestGroup = SchedGroupMembers.idTestGroup\n' + ' AND TestGroups.tsExpire < %s\n' + ' AND TestGroups.tsEffective >= %s\n' + 'ORDER BY SchedGroupMembers.iSchedPriority DESC, SchedGroupMembers.idTestGroup\n' + , (idSchedGroup, tsEffective, tsEffective, tsEffective, tsEffective, )); + aaoRows = self._oDb.fetchAll(); + aoRet = []; + for aoRow in aaoRows: + aoRet.append(SchedGroupMemberDataEx().initFromDbRow(aoRow)); + return aoRet; + + def getTestCasesForGroup(self, idSchedGroup, cMax = None): + """ + Gets the enabled testcases w/ testgroup+priority for the given scheduling group. + + Returns an array of TestCaseData instances (ordered by group id, descending + testcase priority, and testcase IDs) with an extra iSchedPriority member. + May raise exception on DB error or if the result exceeds cMax. + """ + + self._oDb.execute('SELECT TestGroupMembers.idTestGroup, TestGroupMembers.iSchedPriority, TestCases.*\n' + 'FROM SchedGroupMembers, TestGroups, TestGroupMembers, TestCases\n' + 'WHERE SchedGroupMembers.idSchedGroup = %s\n' + ' AND SchedGroupMembers.tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND TestGroups.idTestGroup = SchedGroupMembers.idTestGroup\n' + ' AND TestGroups.tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND TestGroupMembers.idTestGroup = TestGroups.idTestGroup\n' + ' AND TestGroupMembers.tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND TestCases.idTestCase = TestGroupMembers.idTestCase\n' + ' AND TestCases.tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND TestCases.fEnabled = TRUE\n' + 'ORDER BY TestGroupMembers.idTestGroup, TestGroupMembers.iSchedPriority DESC, TestCases.idTestCase\n' + , (idSchedGroup,)); + + if cMax is not None and self._oDb.getRowCount() > cMax: + raise TMExceptionBase('Too many testcases for scheduling group %s: %s, max %s' + % (idSchedGroup, cMax, self._oDb.getRowCount(),)); + + aoRet = []; + for aoRow in self._oDb.fetchAll(): + oTestCase = TestCaseData().initFromDbRow(aoRow[2:]); + oTestCase.idTestGroup = aoRow[0]; + oTestCase.iSchedPriority = aoRow[1]; + aoRet.append(oTestCase); + return aoRet; + + def getTestCaseArgsForGroup(self, idSchedGroup, cMax = None): + """ + Gets the testcase argument variation w/ testgroup+priority for the given scheduling group. + + Returns an array TestCaseArgsData instance (sorted by group and + variation id) with an extra iSchedPriority member. + May raise exception on DB error or if the result exceeds cMax. + """ + + self._oDb.execute('SELECT TestGroupMembers.idTestGroup, TestGroupMembers.iSchedPriority, TestCaseArgs.*\n' + 'FROM SchedGroupMembers, TestGroups, TestGroupMembers, TestCaseArgs, TestCases\n' + 'WHERE SchedGroupMembers.idSchedGroup = %s\n' + ' AND SchedGroupMembers.tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND TestGroups.idTestGroup = SchedGroupMembers.idTestGroup\n' + ' AND TestGroups.tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND TestGroupMembers.idTestGroup = TestGroups.idTestGroup\n' + ' AND TestGroupMembers.tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND TestCaseArgs.idTestCase = TestGroupMembers.idTestCase\n' + ' AND TestCaseArgs.tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND ( TestGroupMembers.aidTestCaseArgs is NULL\n' + ' OR TestCaseArgs.idTestCaseArgs = ANY(TestGroupMembers.aidTestCaseArgs) )\n' + ' AND TestCases.idTestCase = TestCaseArgs.idTestCase\n' + ' AND TestCases.tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND TestCases.fEnabled = TRUE\n' + 'ORDER BY TestGroupMembers.idTestGroup, TestGroupMembers.idTestCase, TestCaseArgs.idTestCaseArgs\n' + , (idSchedGroup,)); + + if cMax is not None and self._oDb.getRowCount() > cMax: + raise TMExceptionBase('Too many argument variations for scheduling group %s: %s, max %s' + % (idSchedGroup, cMax, self._oDb.getRowCount(),)); + + aoRet = []; + for aoRow in self._oDb.fetchAll(): + oVariation = TestCaseArgsData().initFromDbRow(aoRow[2:]); + oVariation.idTestGroup = aoRow[0]; + oVariation.iSchedPriority = aoRow[1]; + aoRet.append(oVariation); + return aoRet; + + def exists(self, sName): + """Checks if a group with the given name exists.""" + self._oDb.execute('SELECT idSchedGroup\n' + 'FROM SchedGroups\n' + 'WHERE tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND sName = %s\n' + 'LIMIT 1\n' + , (sName,)); + return self._oDb.getRowCount() > 0; + + def getById(self, idSchedGroup): + """Get Scheduling Group data by idSchedGroup""" + self._oDb.execute('SELECT *\n' + 'FROM SchedGroups\n' + 'WHERE tsExpire = \'infinity\'::timestamp\n' + ' AND idSchedGroup = %s;', (idSchedGroup,)) + aRows = self._oDb.fetchAll() + if len(aRows) not in (0, 1): + raise self._oDb.integrityException( + 'Found more than one scheduling groups with the same credentials. Database structure is corrupted.') + try: + return SchedGroupData().initFromDbRow(aRows[0]) + except IndexError: + return None + + + # + # Internal helpers. + # + + def _assertUnique(self, sName, idSchedGroupIgnore = None): + """ + Checks that the scheduling group name is unique. + Raises exception if the name is already in use. + """ + if idSchedGroupIgnore is None: + self._oDb.execute('SELECT idSchedGroup\n' + 'FROM SchedGroups\n' + 'WHERE tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND sName = %s\n' + , ( sName, ) ); + else: + self._oDb.execute('SELECT idSchedGroup\n' + 'FROM SchedGroups\n' + 'WHERE tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND sName = %s\n' + ' AND idSchedGroup <> %s\n' + , ( sName, idSchedGroupIgnore, ) ); + if self._oDb.getRowCount() > 0: + raise TMRowInUse('Scheduling group name (%s) is already in use.' % (sName,)); + return True; + + def _readdEntry(self, uidAuthor, oData, tsEffective = None): + """ + Re-adds the SchedGroups entry. Used by editEntry and removeEntry. + """ + if tsEffective is None: + tsEffective = self._oDb.getCurrentTimestamp(); + self._oDb.execute('INSERT INTO SchedGroups (\n' + ' uidAuthor,\n' + ' tsEffective,\n' + ' idSchedGroup,\n' + ' sName,\n' + ' sDescription,\n' + ' fEnabled,\n' + ' enmScheduler,\n' + ' idBuildSrc,\n' + ' idBuildSrcTestSuite,\n' + ' sComment )\n' + 'VALUES ( %s, %s, %s, %s, %s, %s, %s, %s, %s, %s )\n' + , ( uidAuthor, + tsEffective, + oData.idSchedGroup, + oData.sName, + oData.sDescription, + oData.fEnabled, + oData.enmScheduler, + oData.idBuildSrc, + oData.idBuildSrcTestSuite, + oData.sComment, )); + return True; + + def _historizeEntry(self, idSchedGroup, tsExpire = None): + """ + Historizes the current entry for the given scheduling group. + """ + if tsExpire is None: + tsExpire = self._oDb.getCurrentTimestamp(); + self._oDb.execute('UPDATE SchedGroups\n' + 'SET tsExpire = %s\n' + 'WHERE idSchedGroup = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , ( tsExpire, idSchedGroup, )); + return True; + + def _addSchedGroupMember(self, uidAuthor, oMember, tsEffective = None): + """ + addEntry worker for adding a scheduling group member. + """ + if tsEffective is None: + tsEffective = self._oDb.getCurrentTimestamp(); + self._oDb.execute('INSERT INTO SchedGroupMembers(\n' + ' idSchedGroup,\n' + ' idTestGroup,\n' + ' tsEffective,\n' + ' uidAuthor,\n' + ' iSchedPriority,\n' + ' bmHourlySchedule,\n' + ' idTestGroupPreReq)\n' + 'VALUES (%s, %s, %s, %s, %s, %s, %s)\n' + , ( oMember.idSchedGroup, + oMember.idTestGroup, + tsEffective, + uidAuthor, + oMember.iSchedPriority, + oMember.bmHourlySchedule, + oMember.idTestGroupPreReq, )); + return True; + + def _removeSchedGroupMember(self, uidAuthor, oMember): + """ + Removes a scheduling group member. + """ + + # Try record who removed it by adding an dummy entry that expires immediately. + (tsCur, tsCurMinusOne) = self._oDb.getCurrentTimestamps(); + if oMember.tsEffective not in (tsCur, tsCurMinusOne): + self._historizeSchedGroupMember(oMember, tsCurMinusOne); + self._addSchedGroupMember(uidAuthor, oMember, tsCurMinusOne); # lazy bird. + self._historizeSchedGroupMember(oMember); + else: + self._historizeSchedGroupMember(oMember); + return True; + + def _historizeSchedGroupMember(self, oMember, tsExpire = None): + """ + Historizes the current entry for the given scheduling group. + """ + if tsExpire is None: + tsExpire = self._oDb.getCurrentTimestamp(); + self._oDb.execute('UPDATE SchedGroupMembers\n' + 'SET tsExpire = %s\n' + 'WHERE idSchedGroup = %s\n' + ' AND idTestGroup = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , ( tsExpire, oMember.idSchedGroup, oMember.idTestGroup, )); + return True; + + # + def _addSchedGroupTestBox(self, uidAuthor, oBoxInGroup, tsEffective = None): + """ + addEntry worker for adding a test box to a scheduling group. + """ + if tsEffective is None: + tsEffective = self._oDb.getCurrentTimestamp(); + self._oDb.execute('INSERT INTO TestBoxesInSchedGroups(\n' + ' idSchedGroup,\n' + ' idTestBox,\n' + ' tsEffective,\n' + ' uidAuthor,\n' + ' iSchedPriority)\n' + 'VALUES (%s, %s, %s, %s, %s)\n' + , ( oBoxInGroup.idSchedGroup, + oBoxInGroup.idTestBox, + tsEffective, + uidAuthor, + oBoxInGroup.iSchedPriority, )); + return True; + + def _removeSchedGroupTestBox(self, uidAuthor, oBoxInGroup): + """ + Removes a testbox from a scheduling group. + """ + + # Try record who removed it by adding an dummy entry that expires immediately. + (tsCur, tsCurMinusOne) = self._oDb.getCurrentTimestamps(); + if oBoxInGroup.tsEffective not in (tsCur, tsCurMinusOne): + self._historizeSchedGroupTestBox(oBoxInGroup, tsCurMinusOne); + self._addSchedGroupTestBox(uidAuthor, oBoxInGroup, tsCurMinusOne); # lazy bird. + self._historizeSchedGroupTestBox(oBoxInGroup); + else: + self._historizeSchedGroupTestBox(oBoxInGroup); + return True; + + def _historizeSchedGroupTestBox(self, oBoxInGroup, tsExpire = None): + """ + Historizes the current entry for the given scheduling group. + """ + if tsExpire is None: + tsExpire = self._oDb.getCurrentTimestamp(); + self._oDb.execute('UPDATE TestBoxesInSchedGroups\n' + 'SET tsExpire = %s\n' + 'WHERE idSchedGroup = %s\n' + ' AND idTestBox = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , ( tsExpire, oBoxInGroup.idSchedGroup, oBoxInGroup.idTestBox, )); + return True; + + + +# +# Unit testing. +# + +# pylint: disable=missing-docstring +class SchedGroupMemberDataTestCase(ModelDataBaseTestCase): + def setUp(self): + self.aoSamples = [SchedGroupMemberData(),]; + +class SchedGroupDataTestCase(ModelDataBaseTestCase): + def setUp(self): + self.aoSamples = [SchedGroupData(),]; + +if __name__ == '__main__': + unittest.main(); + # not reached. + diff --git a/src/VBox/ValidationKit/testmanager/core/schedqueue.py b/src/VBox/ValidationKit/testmanager/core/schedqueue.py new file mode 100755 index 00000000..f49c4ad3 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/schedqueue.py @@ -0,0 +1,153 @@ +# -*- coding: utf-8 -*- +# "$Id: schedqueue.py $" + +""" +Test Manager - Test Case Queue. +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + +## Standard python imports. +#import unittest + +from testmanager.core.base import ModelDataBase, ModelLogicBase, TMExceptionBase #, ModelDataBaseTestCase + + +class SchedQueueEntry(ModelDataBase): + """ + SchedQueue listing entry + + Note! This could be turned into a SchedQueueDataEx class if we start + fetching all the fields from the scheduing queue. + """ + + def __init__(self): + ModelDataBase.__init__(self) + + self.idItem = None + self.tsLastScheduled = None + self.sSchedGroup = None + self.sTestGroup = None + self.sTestCase = None + self.fUpToDate = None + self.iPerSchedGroupRowNumber = None; + + def initFromDbRow(self, aoRow): + """ + Re-initializes the object from a SchedQueueLogic::fetchForListing select. + Returns self. Raises exception if aoRow is None. + """ + if aoRow is None: + raise TMExceptionBase('TestCaseQueue row not found.') + + self.idItem = aoRow[0] + self.tsLastScheduled = aoRow[1] + self.sSchedGroup = aoRow[2] + self.sTestGroup = aoRow[3] + self.sTestCase = aoRow[4] + self.fUpToDate = aoRow[5] + self.iPerSchedGroupRowNumber = aoRow[6]; + return self + + +class SchedQueueLogic(ModelLogicBase): + """ + SchedQueues logic. + """ + def __init__(self, oDb): + ModelLogicBase.__init__(self, oDb) + + def fetchForListing(self, iStart, cMaxRows, tsNow, aiSortColumns = None): + """ + Fetches SchedQueues entries. + + Returns an array (list) of SchedQueueEntry items, empty list if none. + Raises exception on error. + """ + _, _ = tsNow, aiSortColumns + self._oDb.execute(''' +SELECT SchedQueues.idItem, + SchedQueues.tsLastScheduled, + SchedGroups.sName, + TestGroups.sName, + TestCases.sName, + SchedGroups.tsExpire = 'infinity'::TIMESTAMP + AND TestGroups.tsExpire = 'infinity'::TIMESTAMP + AND TestGroups.tsExpire = 'infinity'::TIMESTAMP + AND TestCaseArgs.tsExpire = 'infinity'::TIMESTAMP + AND TestCases.tsExpire = 'infinity'::TIMESTAMP AS fUpToDate, + ROW_NUMBER() OVER (PARTITION BY SchedQueues.idSchedGroup + ORDER BY SchedQueues.tsLastScheduled, + SchedQueues.idItem) AS iPerSchedGroupRowNumber +FROM SchedQueues + INNER JOIN SchedGroups + ON SchedGroups.idSchedGroup = SchedQueues.idSchedGroup + AND SchedGroups.tsExpire > SchedQueues.tsConfig + AND SchedGroups.tsEffective <= SchedQueues.tsConfig + INNER JOIN TestGroups + ON TestGroups.idTestGroup = SchedQueues.idTestGroup + AND TestGroups.tsExpire > SchedQueues.tsConfig + AND TestGroups.tsEffective <= SchedQueues.tsConfig + INNER JOIN TestCaseArgs + ON TestCaseArgs.idGenTestCaseArgs = SchedQueues.idGenTestCaseArgs + INNER JOIN TestCases + ON TestCases.idTestCase = TestCaseArgs.idTestCase + AND TestCases.tsExpire > SchedQueues.tsConfig + AND TestCases.tsEffective <= SchedQueues.tsConfig +ORDER BY iPerSchedGroupRowNumber, + SchedGroups.sName DESC +LIMIT %s OFFSET %s''' % (cMaxRows, iStart,)) + aoRows = [] + for _ in range(self._oDb.getRowCount()): + aoRows.append(SchedQueueEntry().initFromDbRow(self._oDb.fetchOne())) + return aoRows + +# +# Unit testing. +# + +## @todo SchedQueueEntry isn't a typical ModelDataBase child (not fetching all +## fields; is an extended data class mixing data from multiple tables), so +## this won't work yet. +# +## pylint: disable=missing-docstring +#class TestCaseQueueDataTestCase(ModelDataBaseTestCase): +# def setUp(self): +# self.aoSamples = [SchedQueueEntry(),] +# +# +#if __name__ == '__main__': +# unittest.main() +# # not reached. +# diff --git a/src/VBox/ValidationKit/testmanager/core/schedulerbase.py b/src/VBox/ValidationKit/testmanager/core/schedulerbase.py new file mode 100755 index 00000000..c28b43cf --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/schedulerbase.py @@ -0,0 +1,1570 @@ +# -*- coding: utf-8 -*- +# $Id: schedulerbase.py $ +# pylint: disable=too-many-lines + + +""" +Test Manager - Base class and utilities for the schedulers. +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Standard python imports. +import sys; +import unittest; + +# Validation Kit imports. +from common import utils, constants; +from testmanager import config; +from testmanager.core.build import BuildDataEx, BuildLogic; +from testmanager.core.base import ModelDataBase, ModelDataBaseTestCase, TMExceptionBase; +from testmanager.core.buildsource import BuildSourceData, BuildSourceLogic; +from testmanager.core.globalresource import GlobalResourceLogic; +from testmanager.core.schedgroup import SchedGroupData, SchedGroupLogic; +from testmanager.core.systemlog import SystemLogData, SystemLogLogic; +from testmanager.core.testbox import TestBoxData, TestBoxDataEx; +from testmanager.core.testboxstatus import TestBoxStatusData, TestBoxStatusLogic; +from testmanager.core.testcase import TestCaseLogic; +from testmanager.core.testcaseargs import TestCaseArgsDataEx, TestCaseArgsLogic; +from testmanager.core.testset import TestSetData, TestSetLogic; + +# Python 3 hacks: +if sys.version_info[0] >= 3: + xrange = range; # pylint: disable=redefined-builtin,invalid-name + + + +class ReCreateQueueData(object): + """ + Data object for recreating a scheduling queue. + + It's mostly a storage object, but has a few data checking operation + associated with it. + """ + + def __init__(self, oDb, idSchedGroup): + # + # Load data from the database. + # + oSchedGroupLogic = SchedGroupLogic(oDb); + self.oSchedGroup = oSchedGroupLogic.cachedLookup(idSchedGroup); + + # Will extend the entries with aoTestCases and dTestCases members + # further down (SchedGroupMemberDataEx). checkForGroupDepCycles + # will add aidTestGroupPreReqs. + self.aoTestGroups = oSchedGroupLogic.getMembers(idSchedGroup); + + # aoTestCases entries are TestCaseData instance with iSchedPriority + # and idTestGroup added for our purposes. + # We will add oTestGroup and aoArgsVariations members to each further down. + self.aoTestCases = oSchedGroupLogic.getTestCasesForGroup(idSchedGroup, cMax = 4096); + + # Load dependencies. + oTestCaseLogic = TestCaseLogic(oDb) + for oTestCase in self.aoTestCases: + oTestCase.aidPreReqs = oTestCaseLogic.getTestCasePreReqIds(oTestCase.idTestCase, cMax = 4096); + + # aoTestCases entries are TestCaseArgsData instance with iSchedPriority + # and idTestGroup added for our purposes. + # We will add oTestGroup and oTestCase members to each further down. + self.aoArgsVariations = oSchedGroupLogic.getTestCaseArgsForGroup(idSchedGroup, cMax = 65536); + + # + # Generate global lookups. + # + + # Generate a testcase lookup dictionary for use when working on + # argument variations. + self.dTestCases = {}; + for oTestCase in self.aoTestCases: + self.dTestCases[oTestCase.idTestCase] = oTestCase; + assert len(self.dTestCases) <= len(self.aoTestCases); # Note! Can be shorter! + + # Generate a testgroup lookup dictionary. + self.dTestGroups = {}; + for oTestGroup in self.aoTestGroups: + self.dTestGroups[oTestGroup.idTestGroup] = oTestGroup; + assert len(self.dTestGroups) == len(self.aoTestGroups); + + # + # Associate extra members with the base data. + # + if self.aoTestGroups: + # Prep the test groups. + for oTestGroup in self.aoTestGroups: + oTestGroup.aoTestCases = []; + oTestGroup.dTestCases = {}; + + # Link testcases to their group, both directions. Prep testcases for + # argument varation association. + oTestGroup = self.aoTestGroups[0]; + for oTestCase in self.aoTestCases: + if oTestGroup.idTestGroup != oTestCase.idTestGroup: + oTestGroup = self.dTestGroups[oTestCase.idTestGroup]; + + assert oTestCase.idTestCase not in oTestGroup.dTestCases; + oTestGroup.dTestCases[oTestCase.idTestCase] = oTestCase; + oTestGroup.aoTestCases.append(oTestCase); + oTestCase.oTestGroup = oTestGroup; + oTestCase.aoArgsVariations = []; + + # Associate testcase argument variations with their testcases (group) + # in both directions. + oTestGroup = self.aoTestGroups[0]; + oTestCase = self.aoTestCases[0] if self.aoTestCases else None; + for oArgVariation in self.aoArgsVariations: + if oTestGroup.idTestGroup != oArgVariation.idTestGroup: + oTestGroup = self.dTestGroups[oArgVariation.idTestGroup]; + if oTestCase.idTestCase != oArgVariation.idTestCase or oTestCase.idTestGroup != oArgVariation.idTestGroup: + oTestCase = oTestGroup.dTestCases[oArgVariation.idTestCase]; + + oTestCase.aoArgsVariations.append(oArgVariation); + oArgVariation.oTestCase = oTestCase; + oArgVariation.oTestGroup = oTestGroup; + + else: + assert not self.aoTestCases; + assert not self.aoArgsVariations; + # done. + + @staticmethod + def _addPreReqError(aoErrors, aidChain, oObj, sMsg): + """ Returns a chain of IDs error entry. """ + + sMsg += ' Dependency chain: %s' % (aidChain[0],); + for i in range(1, len(aidChain)): + sMsg += ' -> %s' % (aidChain[i],); + + aoErrors.append([sMsg, oObj]); + return aoErrors; + + def checkForGroupDepCycles(self): + """ + Checks for testgroup depencency cycles and any missing testgroup + dependencies. + Returns array of errors (see SchedulderBase.recreateQueue()). + """ + aoErrors = []; + for oTestGroup in self.aoTestGroups: + idPreReq = oTestGroup.idTestGroupPreReq; + if idPreReq is None: + oTestGroup.aidTestGroupPreReqs = []; + continue; + + aidChain = [oTestGroup.idTestGroup,]; + while idPreReq is not None: + aidChain.append(idPreReq); + if len(aidChain) >= 10: + self._addPreReqError(aoErrors, aidChain, oTestGroup, + 'TestGroup #%s prerequisite chain is too long!' + % (oTestGroup.idTestGroup,)); + break; + + oDep = self.dTestGroups.get(idPreReq, None); + if oDep is None: + self._addPreReqError(aoErrors, aidChain, oTestGroup, + 'TestGroup #%s prerequisite #%s is not in the scheduling group!' + % (oTestGroup.idTestGroup, idPreReq,)); + break; + + idPreReq = oDep.idTestGroupPreReq; + oTestGroup.aidTestGroupPreReqs = aidChain[1:]; + + return aoErrors; + + + def checkForMissingTestCaseDeps(self): + """ + Checks that testcase dependencies stays within bounds. We do not allow + dependencies outside a testgroup, no dependency cycles or even remotely + long dependency chains. + + Returns array of errors (see SchedulderBase.recreateQueue()). + """ + aoErrors = []; + for oTestGroup in self.aoTestGroups: + for oTestCase in oTestGroup.aoTestCases: + if not oTestCase.aidPreReqs: + continue; + + # Stupid recursion code using special stack(s). + aiIndexes = [[oTestCase, 0], ]; + aidChain = [oTestCase.idTestGroup,]; + while aiIndexes: + (oCur, i) = aiIndexes[-1]; + if i >= len(oCur.aidPreReqs): + aiIndexes.pop(); + aidChain.pop(); + else: + aiIndexes[-1][1] = i + 1; # whatever happens, we'll advance on the current level. + + idPreReq = oTestCase.aidPreReqs[i]; + oDep = oTestGroup.dTestCases.get(idPreReq, None); + if oDep is None: + self._addPreReqError(aoErrors, aidChain, oTestCase, + 'TestCase #%s prerequisite #%s is not in the scheduling group!' + % (oTestCase.idTestCase, idPreReq)); + elif idPreReq in aidChain: + self._addPreReqError(aoErrors, aidChain, oTestCase, + 'TestCase #%s prerequisite #%s creates a cycle!' + % (oTestCase.idTestCase, idPreReq)); + elif not oDep.aiPreReqs: + pass; + elif len(aidChain) >= 10: + self._addPreReqError(aoErrors, aidChain, oTestCase, + 'TestCase #%s prerequisite chain is too long!' % (oTestCase.idTestCase,)); + else: + aiIndexes.append([oDep, 0]); + aidChain.append(idPreReq); + + return aoErrors; + + def deepTestGroupSort(self): + """ + Sorts the testgroups and their testcases by priority and dependencies. + Note! Don't call this before checking for dependency cycles! + """ + if not self.aoTestGroups: + return; + + # + # ASSUMES groups as well as testcases are sorted by priority by the + # database. So we only have to concern ourselves with the dependency + # sorting. + # + iGrpPrio = self.aoTestGroups[0].iSchedPriority; + for iTestGroup, oTestGroup in enumerate(self.aoTestGroups): + if oTestGroup.iSchedPriority > iGrpPrio: + raise TMExceptionBase('Incorrectly sorted testgroups returned by database: iTestGroup=%s prio=%s %s' + % ( iTestGroup, iGrpPrio, + ', '.join(['(%s: %s)' % (oCur.idTestGroup, oCur.iSchedPriority) + for oCur in self.aoTestGroups]), ) ); + iGrpPrio = oTestGroup.iSchedPriority; + + if oTestGroup.aoTestCases: + iTstPrio = oTestGroup.aoTestCases[0].iSchedPriority; + for iTestCase, oTestCase in enumerate(oTestGroup.aoTestCases): + if oTestCase.iSchedPriority > iTstPrio: + raise TMExceptionBase('Incorrectly sorted testcases returned by database: i=%s prio=%s idGrp=%s %s' + % ( iTestCase, iTstPrio, oTestGroup.idTestGroup, + ', '.join(['(%s: %s)' % (oCur.idTestCase, oCur.iSchedPriority) + for oCur in oTestGroup.aoTestCases]),)); + + # + # Sort the testgroups by dependencies. + # + i = 0; + while i < len(self.aoTestGroups): + oTestGroup = self.aoTestGroups[i]; + if oTestGroup.idTestGroupPreReq is not None: + iPreReq = self.aoTestGroups.index(self.dTestGroups[oTestGroup.idTestGroupPreReq]); + if iPreReq > i: + # The prerequisite is after the current entry. Move the + # current entry so that it's following it's prereq entry. + self.aoTestGroups.insert(iPreReq + 1, oTestGroup); + self.aoTestGroups.pop(i); + continue; + assert iPreReq < i; + i += 1; # Advance. + + # + # Sort the testcases by dependencies. + # Same algorithm as above, just more prerequisites. + # + for oTestGroup in self.aoTestGroups: + i = 0; + while i < len(oTestGroup.aoTestCases): + oTestCase = oTestGroup.aoTestCases[i]; + if oTestCase.aidPreReqs: + for idPreReq in oTestCase.aidPreReqs: + iPreReq = oTestGroup.aoTestCases.index(oTestGroup.dTestCases[idPreReq]); + if iPreReq > i: + # The prerequisite is after the current entry. Move the + # current entry so that it's following it's prereq entry. + oTestGroup.aoTestGroups.insert(iPreReq + 1, oTestCase); + oTestGroup.aoTestGroups.pop(i); + i -= 1; # Don't advance. + break; + assert iPreReq < i; + i += 1; # Advance. + + + +class SchedQueueData(ModelDataBase): + """ + Scheduling queue data item. + """ + + ksIdAttr = 'idSchedGroup'; + + ksParam_idSchedGroup = 'SchedQueueData_idSchedGroup'; + ksParam_idItem = 'SchedQueueData_idItem'; + ksParam_offQueue = 'SchedQueueData_offQueue'; + ksParam_idGenTestCaseArgs = 'SchedQueueData_idGenTestCaseArgs'; + ksParam_idTestGroup = 'SchedQueueData_idTestGroup'; + ksParam_aidTestGroupPreReqs = 'SchedQueueData_aidTestGroupPreReqs'; + ksParam_bmHourlySchedule = 'SchedQueueData_bmHourlySchedule'; + ksParam_tsConfig = 'SchedQueueData_tsConfig'; + ksParam_tsLastScheduled = 'SchedQueueData_tsLastScheduled'; + ksParam_idTestSetGangLeader = 'SchedQueueData_idTestSetGangLeader'; + ksParam_cMissingGangMembers = 'SchedQueueData_cMissingGangMembers'; + + kasAllowNullAttributes = [ 'idItem', 'offQueue', 'aidTestGroupPreReqs', 'bmHourlySchedule', 'idTestSetGangLeader', + 'tsConfig', 'tsLastScheduled' ]; + + + def __init__(self): + ModelDataBase.__init__(self); + + # + # Initialize with defaults. + # See the database for explanations of each of these fields. + # + self.idSchedGroup = None; + self.idItem = None; + self.offQueue = None; + self.idGenTestCaseArgs = None; + self.idTestGroup = None; + self.aidTestGroupPreReqs = None; + self.bmHourlySchedule = None; + self.tsConfig = None; + self.tsLastScheduled = None; + self.idTestSetGangLeader = None; + self.cMissingGangMembers = 1; + + def initFromValues(self, idSchedGroup, idGenTestCaseArgs, idTestGroup, aidTestGroupPreReqs, # pylint: disable=too-many-arguments + bmHourlySchedule, cMissingGangMembers, + idItem = None, offQueue = None, tsConfig = None, tsLastScheduled = None, idTestSetGangLeader = None): + """ + Reinitialize with all attributes potentially given as inputs. + Return self. + """ + self.idSchedGroup = idSchedGroup; + self.idItem = idItem; + self.offQueue = offQueue; + self.idGenTestCaseArgs = idGenTestCaseArgs; + self.idTestGroup = idTestGroup; + self.aidTestGroupPreReqs = aidTestGroupPreReqs; + self.bmHourlySchedule = bmHourlySchedule; + self.tsConfig = tsConfig; + self.tsLastScheduled = tsLastScheduled; + self.idTestSetGangLeader = idTestSetGangLeader; + self.cMissingGangMembers = cMissingGangMembers; + return self; + + def initFromDbRow(self, aoRow): + """ + Initialize from database row (SELECT * FROM SchedQueues). + Returns self. + Raises exception if no row is specfied. + """ + if aoRow is None: + raise TMExceptionBase('SchedQueueData not found.'); + + self.idSchedGroup = aoRow[0]; + self.idItem = aoRow[1]; + self.offQueue = aoRow[2]; + self.idGenTestCaseArgs = aoRow[3]; + self.idTestGroup = aoRow[4]; + self.aidTestGroupPreReqs = aoRow[5]; + self.bmHourlySchedule = aoRow[6]; + self.tsConfig = aoRow[7]; + self.tsLastScheduled = aoRow[8]; + self.idTestSetGangLeader = aoRow[9]; + self.cMissingGangMembers = aoRow[10]; + return self; + + + + + + +class SchedulerBase(object): + """ + The scheduler base class. + + The scheduler classes have two functions: + 1. Recreate the scheduling queue. + 2. Pick the next task from the queue. + + The first is scheduler specific, the latter isn't. + """ + + class BuildCache(object): + """ Build cache. """ + + class BuildCacheIterator(object): + """ Build class iterator. """ + def __init__(self, oCache): + self.oCache = oCache; + self.iCur = 0; + + def __iter__(self): + """Returns self, required by the language.""" + return self; + + def __next__(self): + """Returns the next build, raises StopIteration when the end has been reached.""" + while True: + if self.iCur >= len(self.oCache.aoEntries): + oEntry = self.oCache.fetchFromCursor(); + if oEntry is None: + raise StopIteration; + else: + oEntry = self.oCache.aoEntries[self.iCur]; + self.iCur += 1; + if not oEntry.fRemoved: + return oEntry; + return None; # not reached, but make pylint happy (for now). + + def next(self): + """ For python 2.x. """ + return self.__next__(); + + class BuildCacheEntry(object): + """ Build cache entry. """ + + def __init__(self, oBuild, fMaybeBlacklisted): + self.oBuild = oBuild; + self._fBlacklisted = None if fMaybeBlacklisted is True else False; + self.fRemoved = False; + self._dPreReqDecisions = {}; + + def remove(self): + """ + Marks the cache entry as removed. + This doesn't actually remove it from the cache array, only marks + it as removed. It has no effect on open iterators. + """ + self.fRemoved = True; + + def getPreReqDecision(self, sPreReqSet): + """ + Retrieves a cached prerequisite decision. + Returns boolean if found, None if not. + """ + return self._dPreReqDecisions.get(sPreReqSet); + + def setPreReqDecision(self, sPreReqSet, fDecision): + """ + Caches a prerequistie decision. + """ + self._dPreReqDecisions[sPreReqSet] = fDecision; + return fDecision; + + def isBlacklisted(self, oDb): + """ Checks if the build is blacklisted. """ + if self._fBlacklisted is None: + self._fBlacklisted = BuildLogic(oDb).isBuildBlacklisted(self.oBuild); + return self._fBlacklisted; + + + def __init__(self): + self.aoEntries = []; + self.oCursor = None; + + def setupSource(self, oDb, idBuildSrc, sOs, sCpuArch, tsNow): + """ Configures the build cursor for the cache. """ + if not self.aoEntries and self.oCursor is None: + oBuildSource = BuildSourceData().initFromDbWithId(oDb, idBuildSrc, tsNow); + self.oCursor = BuildSourceLogic(oDb).openBuildCursor(oBuildSource, sOs, sCpuArch, tsNow); + return True; + + def __iter__(self): + """Return an iterator.""" + return self.BuildCacheIterator(self); + + def fetchFromCursor(self): + """ Fetches a build from the cursor and adds it to the cache.""" + if self.oCursor is None: + return None; + + try: + aoRow = self.oCursor.fetchOne(); + except: + return None; + if aoRow is None: + return None; + + oBuild = BuildDataEx().initFromDbRow(aoRow); + oEntry = self.BuildCacheEntry(oBuild, aoRow[-1]); + self.aoEntries.append(oEntry); + return oEntry; + + def __init__(self, oDb, oSchedGrpData, iVerbosity = 0, tsSecStart = None): + self._oDb = oDb; + self._oSchedGrpData = oSchedGrpData; + self._iVerbosity = iVerbosity; + self._asMessages = []; + self._tsSecStart = tsSecStart if tsSecStart is not None else utils.timestampSecond(); + self.oBuildCache = self.BuildCache(); + self.dTestGroupMembers = {}; + + @staticmethod + def _instantiate(oDb, oSchedGrpData, iVerbosity = 0, tsSecStart = None): + """ + Instantiate the scheduler specified by the scheduling group. + Returns scheduler child class instance. May raise exception if + the input is invalid. + """ + if oSchedGrpData.enmScheduler == SchedGroupData.ksScheduler_BestEffortContinuousIntegration: + from testmanager.core.schedulerbeci import SchdulerBeci; + oScheduler = SchdulerBeci(oDb, oSchedGrpData, iVerbosity, tsSecStart); + else: + raise oDb.integrityException('Invalid scheduler "%s", idSchedGroup=%d' \ + % (oSchedGrpData.enmScheduler, oSchedGrpData.idSchedGroup)); + return oScheduler; + + + # + # Misc. + # + + def msgDebug(self, sText): + """Debug printing.""" + if self._iVerbosity > 1: + self._asMessages.append('debug:' + sText); + return None; + + def msgInfo(self, sText): + """Info printing.""" + if self._iVerbosity > 1: + self._asMessages.append('info: ' + sText); + return None; + + def dprint(self, sMsg): + """Prints a debug message to the srv glue log (see config.py). """ + if config.g_kfSrvGlueDebugScheduler: + self._oDb.dprint(sMsg); + return None; + + def getElapsedSecs(self): + """ Returns the number of seconds this scheduling task has been running. """ + tsSecNow = utils.timestampSecond(); + if tsSecNow < self._tsSecStart: # paranoia + self._tsSecStart = tsSecNow; + return tsSecNow - self._tsSecStart; + + + # + # Create schedule. + # + + def _recreateQueueCancelGatherings(self): + """ + Cancels all pending gang gatherings on the current queue. + """ + self._oDb.execute('SELECT idTestSetGangLeader\n' + 'FROM SchedQueues\n' + 'WHERE idSchedGroup = %s\n' + ' AND idTestSetGangLeader is not NULL\n' + , (self._oSchedGrpData.idSchedGroup,)); + if self._oDb.getRowCount() > 0: + oTBStatusLogic = TestBoxStatusLogic(self._oDb); + for aoRow in self._oDb.fetchAll(): + idTestSetGangLeader = aoRow[0]; + oTBStatusLogic.updateGangStatus(idTestSetGangLeader, + TestBoxStatusData.ksTestBoxState_GangGatheringTimedOut, + fCommit = False); + return True; + + def _recreateQueueItems(self, oData): + """ + Returns an array of queue items (SchedQueueData). + Child classes must override this. + """ + _ = oData; + return []; + + def recreateQueueWorker(self): + """ + Worker for recreateQueue. + """ + + # + # Collect the necessary data and validate it. + # + oData = ReCreateQueueData(self._oDb, self._oSchedGrpData.idSchedGroup); + aoErrors = oData.checkForGroupDepCycles(); + aoErrors.extend(oData.checkForMissingTestCaseDeps()); + if not aoErrors: + oData.deepTestGroupSort(); + + # + # The creation of the scheduling queue is done by the child class. + # + # We will try guess where in queue we're currently at and rotate + # the items such that we will resume execution in the approximately + # same position. The goal of the scheduler is to provide a 100% + # deterministic result so that if we regenerate the queue when there + # are no changes to the testcases, testgroups or scheduling groups + # involved, test execution will be unchanged (save for maybe just a + # little for gang gathering). + # + aoItems = []; + if not oData.oSchedGroup.fEnabled: + self.msgInfo('Disabled.'); + elif not oData.aoArgsVariations: + self.msgInfo('Found no test case argument variations.'); + else: + aoItems = self._recreateQueueItems(oData); + self.msgDebug('len(aoItems)=%s' % (len(aoItems),)); + #for i in range(len(aoItems)): + # self.msgDebug('aoItems[%2d]=%s' % (i, aoItems[i])); + if aoItems: + self._oDb.execute('SELECT offQueue FROM SchedQueues WHERE idSchedGroup = %s ORDER BY idItem LIMIT 1' + , (self._oSchedGrpData.idSchedGroup,)); + if self._oDb.getRowCount() > 0: + offQueue = self._oDb.fetchOne()[0]; + self._oDb.execute('SELECT COUNT(*) FROM SchedQueues WHERE idSchedGroup = %s' + , (self._oSchedGrpData.idSchedGroup,)); + cItems = self._oDb.fetchOne()[0]; + offQueueNew = (offQueue * cItems) // len(aoItems); + if offQueueNew != 0: + aoItems = aoItems[offQueueNew:] + aoItems[:offQueueNew]; + + # + # Replace the scheduling queue. + # Care need to be take to first timeout/abort any gangs in the + # gathering state since these use the queue to set up the date. + # + self._recreateQueueCancelGatherings(); + self._oDb.execute('DELETE FROM SchedQueues WHERE idSchedGroup = %s\n', (self._oSchedGrpData.idSchedGroup,)); + if aoItems: + self._oDb.insertList('INSERT INTO SchedQueues (\n' + ' idSchedGroup,\n' + ' offQueue,\n' + ' idGenTestCaseArgs,\n' + ' idTestGroup,\n' + ' aidTestGroupPreReqs,\n' + ' bmHourlySchedule,\n' + ' cMissingGangMembers )\n', + aoItems, self._formatItemForInsert); + return (aoErrors, self._asMessages); + + def _formatItemForInsert(self, oItem): + """ + Used by recreateQueueWorker together with TMDatabaseConnect::insertList + """ + return self._oDb.formatBindArgs('(%s,%s,%s,%s,%s,%s,%s)' + , ( oItem.idSchedGroup, + oItem.offQueue, + oItem.idGenTestCaseArgs, + oItem.idTestGroup, + oItem.aidTestGroupPreReqs if oItem.aidTestGroupPreReqs else None, + oItem.bmHourlySchedule, + oItem.cMissingGangMembers + )); + + @staticmethod + def recreateQueue(oDb, uidAuthor, idSchedGroup, iVerbosity = 1): + """ + (Re-)creates the scheduling queue for the given group. + + Returns (asMessages, asMessages). On success the array with the error + will be empty, on failure it will contain (sError, oRelatedObject) + entries. The messages is for debugging and are simple strings. + + Raises exception database error. + """ + + aoExtraMsgs = []; + if oDb.debugIsExplainEnabled(): + aoExtraMsgs += ['Warning! Disabling SQL explain to avoid deadlocking against locked tables.']; + oDb.debugDisableExplain(); + + aoErrors = []; + asMessages = []; + try: + # + # To avoid concurrency issues (SchedQueues) and inconsistent data (*), + # we lock quite a few tables while doing this work. We access more + # data than scheduleNewTask so we lock some additional tables. + # + oDb.rollback(); + oDb.begin(); + oDb.execute('LOCK TABLE SchedGroups, SchedGroupMembers, TestGroups, TestGroupMembers IN SHARE MODE'); + oDb.execute('LOCK TABLE TestBoxes, TestCaseArgs, TestCases IN SHARE MODE'); + oDb.execute('LOCK TABLE TestBoxStatuses, SchedQueues IN EXCLUSIVE MODE'); + + # + # Instantiate the scheduler and call the worker function. + # + oSchedGrpData = SchedGroupData().initFromDbWithId(oDb, idSchedGroup); + oScheduler = SchedulerBase._instantiate(oDb, oSchedGrpData, iVerbosity); + + (aoErrors, asMessages) = oScheduler.recreateQueueWorker(); + if not aoErrors: + SystemLogLogic(oDb).addEntry(SystemLogData.ksEvent_SchedQueueRecreate, + 'User #%d recreated sched queue #%d.' % (uidAuthor, idSchedGroup,)); + oDb.commit(); + else: + oDb.rollback(); + + except: + oDb.rollback(); + raise; + + return (aoErrors, aoExtraMsgs + asMessages); + + + @staticmethod + def cleanUpOrphanedQueues(oDb): + """ + Removes orphan scheduling queues from the SchedQueues table. + + Queues becomes orphaned when the scheduling group they belongs to has been deleted. + + Returns number of orphaned queues. + Raises exception database error. + """ + cRet = 0; + try: + oDb.rollback(); + oDb.begin(); + oDb.execute(''' +SELECT SchedQueues.idSchedGroup +FROM SchedQueues + LEFT OUTER JOIN SchedGroups + ON SchedGroups.idSchedGroup = SchedQueues.idSchedGroup + AND SchedGroups.tsExpire = 'infinity'::TIMESTAMP +WHERE SchedGroups.idSchedGroup is NULL +GROUP BY SchedQueues.idSchedGroup'''); + aaoOrphanRows = oDb.fetchAll(); + cRet = len(aaoOrphanRows); + if cRet > 0: + oDb.execute('DELETE FROM SchedQueues WHERE idSchedGroup IN (%s)' + % (','.join([str(aoRow[0]) for aoRow in aaoOrphanRows]),)); + oDb.commit(); + except: + oDb.rollback(); + raise; + return cRet; + + + # + # Schedule Task. + # + + def _composeGangArguments(self, idTestSet): + """ + Composes the gang specific testdriver arguments. + Returns command line string, including a leading space. + """ + + oTestSet = TestSetData().initFromDbWithId(self._oDb, idTestSet); + aoGangMembers = TestSetLogic(self._oDb).getGang(oTestSet.idTestSetGangLeader); + + sArgs = ' --gang-member-no %s --gang-members %s' % (oTestSet.iGangMemberNo, len(aoGangMembers)); + for i, _ in enumerate(aoGangMembers): + sArgs = ' --gang-ipv4-%s %s' % (i, aoGangMembers[i].ip); ## @todo IPv6 + + return sArgs; + + + def composeExecResponseWorker(self, idTestSet, oTestEx, oTestBox, oBuild, oValidationKitBuild, sBaseUrl): + """ + Given all the bits of data, compose an EXEC command response to the testbox. + """ + sScriptZips = oTestEx.oTestCase.sValidationKitZips; + if sScriptZips is None or sScriptZips.find('@VALIDATIONKIT_ZIP@') >= 0: + assert oValidationKitBuild; + if sScriptZips is None: + sScriptZips = oValidationKitBuild.sBinaries; + else: + sScriptZips = sScriptZips.replace('@VALIDATIONKIT_ZIP@', oValidationKitBuild.sBinaries); + sScriptZips = sScriptZips.replace('@DOWNLOAD_BASE_URL@', sBaseUrl + config.g_ksTmDownloadBaseUrlRel); + + sCmdLine = oTestEx.oTestCase.sBaseCmd + ' ' + oTestEx.sArgs; + sCmdLine = sCmdLine.replace('@BUILD_BINARIES@', oBuild.sBinaries); + sCmdLine = sCmdLine.strip(); + if oTestEx.cGangMembers > 1: + sCmdLine += ' ' + self._composeGangArguments(idTestSet); + + cSecTimeout = oTestEx.cSecTimeout if oTestEx.cSecTimeout is not None else oTestEx.oTestCase.cSecTimeout; + cSecTimeout = cSecTimeout * oTestBox.pctScaleTimeout // 100; + + dResponse = \ + { + constants.tbresp.ALL_PARAM_RESULT: constants.tbresp.CMD_EXEC, + constants.tbresp.EXEC_PARAM_RESULT_ID: idTestSet, + constants.tbresp.EXEC_PARAM_SCRIPT_ZIPS: sScriptZips, + constants.tbresp.EXEC_PARAM_SCRIPT_CMD_LINE: sCmdLine, + constants.tbresp.EXEC_PARAM_TIMEOUT: cSecTimeout, + }; + return dResponse; + + @staticmethod + def composeExecResponse(oDb, idTestSet, sBaseUrl, iVerbosity = 0): + """ + Composes an EXEC response for a gang member (other than the last). + Returns a EXEC response or raises an exception (DB/input error). + """ + # + # Gather the necessary data. + # + oTestSet = TestSetData().initFromDbWithId(oDb, idTestSet); + oTestBox = TestBoxData().initFromDbWithGenId(oDb, oTestSet.idGenTestBox); + oTestEx = TestCaseArgsDataEx().initFromDbWithGenIdEx(oDb, oTestSet.idGenTestCaseArgs, + tsConfigEff = oTestSet.tsConfig, + tsRsrcEff = oTestSet.tsConfig); + oBuild = BuildDataEx().initFromDbWithId(oDb, oTestSet.idBuild); + oValidationKitBuild = None; + if oTestSet.idBuildTestSuite is not None: + oValidationKitBuild = BuildDataEx().initFromDbWithId(oDb, oTestSet.idBuildTestSuite); + + # + # Instantiate the specified scheduler and let it do the rest. + # + oSchedGrpData = SchedGroupData().initFromDbWithId(oDb, oTestSet.idSchedGroup, oTestSet.tsCreated); + assert oSchedGrpData.fEnabled is True; + assert oSchedGrpData.idBuildSrc is not None; + oScheduler = SchedulerBase._instantiate(oDb, oSchedGrpData, iVerbosity); + + return oScheduler.composeExecResponseWorker(idTestSet, oTestEx, oTestBox, oBuild, oValidationKitBuild, sBaseUrl); + + + def _updateTask(self, oTask, tsNow): + """ + Updates a gang schedule task. + """ + assert oTask.cMissingGangMembers >= 1; + assert oTask.idTestSetGangLeader is not None; + assert oTask.idTestSetGangLeader >= 1; + if tsNow is not None: + self._oDb.execute('UPDATE SchedQueues\n' + ' SET idTestSetGangLeader = %s,\n' + ' cMissingGangMembers = %s,\n' + ' tsLastScheduled = %s\n' + 'WHERE idItem = %s\n' + , (oTask.idTestSetGangLeader, oTask.cMissingGangMembers, tsNow, oTask.idItem,) ); + else: + self._oDb.execute('UPDATE SchedQueues\n' + ' SET cMissingGangMembers = %s\n' + 'WHERE idItem = %s\n' + , (oTask.cMissingGangMembers, oTask.idItem,) ); + return True; + + def _moveTaskToEndOfQueue(self, oTask, cGangMembers, tsNow): + """ + The task has been scheduled successfully, reset it's data move it to + the end of the queue. + """ + if cGangMembers > 1: + self._oDb.execute('UPDATE SchedQueues\n' + ' SET idItem = NEXTVAL(\'SchedQueueItemIdSeq\'),\n' + ' idTestSetGangLeader = NULL,\n' + ' cMissingGangMembers = %s\n' + 'WHERE idItem = %s\n' + , (cGangMembers, oTask.idItem,) ); + else: + self._oDb.execute('UPDATE SchedQueues\n' + ' SET idItem = NEXTVAL(\'SchedQueueItemIdSeq\'),\n' + ' idTestSetGangLeader = NULL,\n' + ' cMissingGangMembers = 1,\n' + ' tsLastScheduled = %s\n' + 'WHERE idItem = %s\n' + , (tsNow, oTask.idItem,) ); + return True; + + + + + def _createTestSet(self, oTask, oTestEx, oTestBoxData, oBuild, oValidationKitBuild, tsNow): + # type: (SchedQueueData, TestCaseArgsDataEx, TestBoxData, BuildDataEx, BuildDataEx, datetime.datetime) -> int + """ + Creates a test set for using the given data. + Will not commit, someone up the callstack will that later on. + + Returns the test set ID, may raise an exception on database error. + """ + # Lazy bird doesn't want to write testset.py and does it all here. + + # + # We're getting the TestSet ID first in order to include it in the base + # file name (that way we can directly relate files on the disk to the + # test set when doing batch work), and also for idTesetSetGangLeader. + # + self._oDb.execute('SELECT NEXTVAL(\'TestSetIdSeq\')'); + idTestSet = self._oDb.fetchOne()[0]; + + sBaseFilename = '%04d/%02d/%02d/%02d/TestSet-%s' \ + % (tsNow.year, tsNow.month, tsNow.day, (tsNow.hour // 6) * 6, idTestSet); + + # + # Gang scheduling parameters. Changes the oTask data for updating by caller. + # + iGangMemberNo = 0; + + if oTestEx.cGangMembers <= 1: + assert oTask.idTestSetGangLeader is None; + assert oTask.cMissingGangMembers <= 1; + elif oTask.idTestSetGangLeader is None: + assert oTask.cMissingGangMembers == oTestEx.cGangMembers; + oTask.cMissingGangMembers = oTestEx.cGangMembers - 1; + oTask.idTestSetGangLeader = idTestSet; + else: + assert oTask.cMissingGangMembers > 0 and oTask.cMissingGangMembers < oTestEx.cGangMembers; + oTask.cMissingGangMembers -= 1; + + # + # Do the database stuff. + # + self._oDb.execute('INSERT INTO TestSets (\n' + ' idTestSet,\n' + ' tsConfig,\n' + ' tsCreated,\n' + ' idBuild,\n' + ' idBuildCategory,\n' + ' idBuildTestSuite,\n' + ' idGenTestBox,\n' + ' idTestBox,\n' + ' idSchedGroup,\n' + ' idTestGroup,\n' + ' idGenTestCase,\n' + ' idTestCase,\n' + ' idGenTestCaseArgs,\n' + ' idTestCaseArgs,\n' + ' sBaseFilename,\n' + ' iGangMemberNo,\n' + ' idTestSetGangLeader )\n' + 'VALUES ( %s,\n' # idTestSet + ' %s,\n' # tsConfig + ' %s,\n' # tsCreated + ' %s,\n' # idBuild + ' %s,\n' # idBuildCategory + ' %s,\n' # idBuildTestSuite + ' %s,\n' # idGenTestBox + ' %s,\n' # idTestBox + ' %s,\n' # idSchedGroup + ' %s,\n' # idTestGroup + ' %s,\n' # idGenTestCase + ' %s,\n' # idTestCase + ' %s,\n' # idGenTestCaseArgs + ' %s,\n' # idTestCaseArgs + ' %s,\n' # sBaseFilename + ' %s,\n' # iGangMemberNo + ' %s)\n' # idTestSetGangLeader + , ( idTestSet, + oTask.tsConfig, + tsNow, + oBuild.idBuild, + oBuild.idBuildCategory, + oValidationKitBuild.idBuild if oValidationKitBuild is not None else None, + oTestBoxData.idGenTestBox, + oTestBoxData.idTestBox, + oTask.idSchedGroup, + oTask.idTestGroup, + oTestEx.oTestCase.idGenTestCase, + oTestEx.oTestCase.idTestCase, + oTestEx.idGenTestCaseArgs, + oTestEx.idTestCaseArgs, + sBaseFilename, + iGangMemberNo, + oTask.idTestSetGangLeader, + )); + + self._oDb.execute('INSERT INTO TestResults (\n' + ' idTestResultParent,\n' + ' idTestSet,\n' + ' tsCreated,\n' + ' idStrName,\n' + ' cErrors,\n' + ' enmStatus,\n' + ' iNestingDepth)\n' + 'VALUES ( NULL,\n' # idTestResultParent + ' %s,\n' # idTestSet + ' %s,\n' # tsCreated + ' 0,\n' # idStrName + ' 0,\n' # cErrors + ' \'running\'::TestStatus_T,\n' + ' 0)\n' # iNestingDepth + 'RETURNING idTestResult' + , ( idTestSet, tsNow, )); + idTestResult = self._oDb.fetchOne()[0]; + + self._oDb.execute('UPDATE TestSets\n' + ' SET idTestResult = %s\n' + 'WHERE idTestSet = %s\n' + , (idTestResult, idTestSet, )); + + return idTestSet; + + def _tryFindValidationKitBit(self, oTestBoxData, tsNow): + """ + Tries to find the most recent validation kit build suitable for the given testbox. + Returns BuildDataEx or None. Raise exception on database error. + + Can be overridden by child classes to change the default build requirements. + """ + oBuildLogic = BuildLogic(self._oDb); + oBuildSource = BuildSourceData().initFromDbWithId(self._oDb, self._oSchedGrpData.idBuildSrcTestSuite, tsNow); + oCursor = BuildSourceLogic(self._oDb).openBuildCursor(oBuildSource, oTestBoxData.sOs, oTestBoxData.sCpuArch, tsNow); + for _ in range(oCursor.getRowCount()): + oBuild = BuildDataEx().initFromDbRow(oCursor.fetchOne()); + if not oBuildLogic.isBuildBlacklisted(oBuild): + return oBuild; + return None; + + def _tryFindBuild(self, oTask, oTestEx, oTestBoxData, tsNow): + """ + Tries to find a fitting build. + Returns BuildDataEx or None. Raise exception on database error. + + Can be overridden by child classes to change the default build requirements. + """ + + # + # Gather the set of prerequisites we have and turn them into a value + # set for use in the loop below. + # + # Note! We're scheduling on testcase level and ignoring argument variation + # selections in TestGroupMembers is intentional. + # + dPreReqs = {}; + + # Direct prerequisites. We assume they're all enabled as this can be + # checked at queue creation time. + for oPreReq in oTestEx.aoTestCasePreReqs: + dPreReqs[oPreReq.idTestCase] = 1; + + # Testgroup dependencies from the scheduling group config. + if oTask.aidTestGroupPreReqs is not None: + for iTestGroup in oTask.aidTestGroupPreReqs: + # Make sure the _active_ test group members are in the cache. + if iTestGroup not in self.dTestGroupMembers: + self._oDb.execute('SELECT DISTINCT TestGroupMembers.idTestCase\n' + 'FROM TestGroupMembers, TestCases\n' + 'WHERE TestGroupMembers.idTestGroup = %s\n' + ' AND TestGroupMembers.tsExpire > %s\n' + ' AND TestGroupMembers.tsEffective <= %s\n' + ' AND TestCases.idTestCase = TestGroupMembers.idTestCase\n' + ' AND TestCases.tsExpire > %s\n' + ' AND TestCases.tsEffective <= %s\n' + ' AND TestCases.fEnabled is TRUE\n' + , (iTestGroup, oTask.tsConfig, oTask.tsConfig, oTask.tsConfig, oTask.tsConfig,)); + aidTestCases = []; + for aoRow in self._oDb.fetchAll(): + aidTestCases.append(aoRow[0]); + self.dTestGroupMembers[iTestGroup] = aidTestCases; + + # Add the testgroup members to the prerequisites. + for idTestCase in self.dTestGroupMembers[iTestGroup]: + dPreReqs[idTestCase] = 1; + + # Create a SQL values table out of them. + sPreReqSet = '' + if dPreReqs: + for idPreReq in sorted(dPreReqs): + sPreReqSet += ', (' + str(idPreReq) + ')'; + sPreReqSet = sPreReqSet[2:]; # drop the leading ', '. + + # + # Try the builds. + # + self.oBuildCache.setupSource(self._oDb, self._oSchedGrpData.idBuildSrc, oTestBoxData.sOs, oTestBoxData.sCpuArch, tsNow); + for oEntry in self.oBuildCache: + # + # Check build requirements set by the test. + # + if not oTestEx.matchesBuildProps(oEntry.oBuild): + continue; + + if oEntry.isBlacklisted(self._oDb): + oEntry.remove(); + continue; + + # + # Check prerequisites. The default scheduler is satisfied if one + # argument variation has been executed successfully. It is not + # satisfied if there are any failure runs. + # + if sPreReqSet: + fDecision = oEntry.getPreReqDecision(sPreReqSet); + if fDecision is None: + # Check for missing prereqs. + self._oDb.execute('SELECT COUNT(*)\n' + 'FROM (VALUES ' + sPreReqSet + ') AS PreReqs(idTestCase)\n' + 'LEFT OUTER JOIN (SELECT idTestSet\n' + ' FROM TestSets\n' + ' WHERE enmStatus IN (%s, %s)\n' + ' AND idBuild = %s\n' + ' ) AS TestSets\n' + ' ON (PreReqs.idTestCase = TestSets.idTestCase)\n' + 'WHERE TestSets.idTestSet is NULL\n' + , ( TestSetData.ksTestStatus_Success, TestSetData.ksTestStatus_Skipped, + oEntry.oBuild.idBuild, )); + cMissingPreReqs = self._oDb.fetchOne()[0]; + if cMissingPreReqs > 0: + self.dprint('build %s is missing %u prerequisites (out of %s)' + % (oEntry.oBuild.idBuild, cMissingPreReqs, sPreReqSet,)); + oEntry.setPreReqDecision(sPreReqSet, False); + continue; + + # Check for failed prereq runs. + self._oDb.execute('SELECT COUNT(*)\n' + 'FROM (VALUES ' + sPreReqSet + ') AS PreReqs(idTestCase),\n' + ' TestSets\n' + 'WHERE PreReqs.idTestCase = TestSets.idTestCase\n' + ' AND TestSets.idBuild = %s\n' + ' AND TestSets.enmStatus IN (%s, %s, %s)\n' + , ( oEntry.oBuild.idBuild, + TestSetData.ksTestStatus_Failure, + TestSetData.ksTestStatus_TimedOut, + TestSetData.ksTestStatus_Rebooted, + ) + ); + cFailedPreReqs = self._oDb.fetchOne()[0]; + if cFailedPreReqs > 0: + self.dprint('build %s is has %u prerequisite failures (out of %s)' + % (oEntry.oBuild.idBuild, cFailedPreReqs, sPreReqSet,)); + oEntry.setPreReqDecision(sPreReqSet, False); + continue; + + oEntry.setPreReqDecision(sPreReqSet, True); + elif not fDecision: + continue; + + # + # If we can, check if the build files still exist. + # + if oEntry.oBuild.areFilesStillThere() is False: + self.dprint('build %s no longer exists' % (oEntry.oBuild.idBuild,)); + oEntry.remove(); + continue; + + self.dprint('found oBuild=%s' % (oEntry.oBuild,)); + return oEntry.oBuild; + return None; + + def _tryFindMatchingBuild(self, oLeaderBuild, oTestBoxData, idBuildSrc): + """ + Tries to find a matching build for gang scheduling. + Returns BuildDataEx or None. Raise exception on database error. + + Can be overridden by child classes to change the default build requirements. + """ + # + # Note! Should probably check build prerequisites if we get a different + # build back, so that we don't use a build which hasn't passed + # the smoke test. + # + _ = idBuildSrc; + return BuildLogic(self._oDb).tryFindSameBuildForOsArch(oLeaderBuild, oTestBoxData.sOs, oTestBoxData.sCpuArch); + + + def _tryAsLeader(self, oTask, oTestEx, oTestBoxData, tsNow, sBaseUrl): + """ + Try schedule the task as a gang leader (can be a gang of one). + Returns response or None. May raise exception on DB error. + """ + + # We don't wait for busy resources, we just try the next test. + oTestArgsLogic = TestCaseArgsLogic(self._oDb); + if not oTestArgsLogic.areResourcesFree(oTestEx): + self.dprint('Cannot get global test resources!'); + return None; + + # + # Find a matching build (this is the difficult bit). + # + oBuild = self._tryFindBuild(oTask, oTestEx, oTestBoxData, tsNow); + if oBuild is None: + self.dprint('No build!'); + return None; + if oTestEx.oTestCase.needValidationKitBit(): + oValidationKitBuild = self._tryFindValidationKitBit(oTestBoxData, tsNow); + if oValidationKitBuild is None: + self.dprint('No validation kit build!'); + return None; + else: + oValidationKitBuild = None; + + # + # Create a testset, allocate the resources and update the state. + # Note! Since resource allocation may still fail, we create a nested + # transaction so we can roll back. (Heed lock warning in docs!) + # + self._oDb.execute('SAVEPOINT tryAsLeader'); + idTestSet = self._createTestSet(oTask, oTestEx, oTestBoxData, oBuild, oValidationKitBuild, tsNow); + + if GlobalResourceLogic(self._oDb).allocateResources(oTestBoxData.idTestBox, oTestEx.aoGlobalRsrc, fCommit = False) \ + is not True: + self._oDb.execute('ROLLBACK TO SAVEPOINT tryAsLeader'); + self.dprint('Failed to allocate global resources!'); + return False; + + if oTestEx.cGangMembers <= 1: + # We're alone, put the task back at the end of the queue and issue EXEC cmd. + self._moveTaskToEndOfQueue(oTask, oTestEx.cGangMembers, tsNow); + dResponse = self.composeExecResponseWorker(idTestSet, oTestEx, oTestBoxData, oBuild, oValidationKitBuild, sBaseUrl); + sTBState = TestBoxStatusData.ksTestBoxState_Testing; + else: + # We're missing gang members, issue WAIT cmd. + self._updateTask(oTask, tsNow if idTestSet == oTask.idTestSetGangLeader else None); + dResponse = { constants.tbresp.ALL_PARAM_RESULT: constants.tbresp.CMD_WAIT, }; + sTBState = TestBoxStatusData.ksTestBoxState_GangGathering; + + TestBoxStatusLogic(self._oDb).updateState(oTestBoxData.idTestBox, sTBState, idTestSet, fCommit = False); + self._oDb.execute('RELEASE SAVEPOINT tryAsLeader'); + return dResponse; + + def _tryAsGangMember(self, oTask, oTestEx, oTestBoxData, tsNow, sBaseUrl): + """ + Try schedule the task as a gang member. + Returns response or None. May raise exception on DB error. + """ + + # + # The leader has choosen a build, we need to find a matching one for our platform. + # (It's up to the scheduler decide upon how strict dependencies are to be enforced + # upon subordinate group members.) + # + oLeaderTestSet = TestSetData().initFromDbWithId(self._oDb, oTestBoxData.idTestSetGangLeader); + + oLeaderBuild = BuildDataEx().initFromDbWithId(self._oDb, oLeaderTestSet.idBuild); + oBuild = self._tryFindMatchingBuild(oLeaderBuild, oTestBoxData, self._oSchedGrpData.idBuildSrc); + if oBuild is None: + return None; + + oValidationKitBuild = None; + if oLeaderTestSet.idBuildTestSuite is not None: + oLeaderValidationKitBit = BuildDataEx().initFromDbWithId(self._oDb, oLeaderTestSet.idBuildTestSuite); + oValidationKitBuild = self._tryFindMatchingBuild(oLeaderValidationKitBit, oTestBoxData, + self._oSchedGrpData.idBuildSrcTestSuite); + + # + # Create a testset and update the state(s). + # + idTestSet = self._createTestSet(oTask, oTestEx, oTestBoxData, oBuild, oValidationKitBuild, tsNow); + + oTBStatusLogic = TestBoxStatusLogic(self._oDb); + if oTask.cMissingGangMembers < 1: + # The whole gang is there, move the task to the end of the queue + # and update the status on the other gang members. + self._moveTaskToEndOfQueue(oTask, oTestEx.cGangMembers, tsNow); + dResponse = self.composeExecResponseWorker(idTestSet, oTestEx, oTestBoxData, oBuild, oValidationKitBuild, sBaseUrl); + sTBState = TestBoxStatusData.ksTestBoxState_GangTesting; + oTBStatusLogic.updateGangStatus(oTask.idTestSetGangLeader, sTBState, fCommit = False); + else: + # We're still missing some gang members, issue WAIT cmd. + self._updateTask(oTask, tsNow if idTestSet == oTask.idTestSetGangLeader else None); + dResponse = { constants.tbresp.ALL_PARAM_RESULT: constants.tbresp.CMD_WAIT, }; + sTBState = TestBoxStatusData.ksTestBoxState_GangGathering; + + oTBStatusLogic.updateState(oTestBoxData.idTestBox, sTBState, idTestSet, fCommit = False); + return dResponse; + + + def scheduleNewTaskWorker(self, oTestBoxData, tsNow, sBaseUrl): + """ + Worker for schduling a new task. + """ + + # + # Iterate the scheduler queue (fetch all to avoid having to concurrent + # queries), trying out each task to see if the testbox can execute it. + # + dRejected = {}; # variations we've already checked out and rejected. + self._oDb.execute('SELECT *\n' + 'FROM SchedQueues\n' + 'WHERE idSchedGroup = %s\n' + ' AND ( bmHourlySchedule IS NULL\n' + ' OR get_bit(bmHourlySchedule, %s) = 1 )\n' + 'ORDER BY idItem ASC\n' + , (self._oSchedGrpData.idSchedGroup, utils.getLocalHourOfWeek()) ); + aaoRows = self._oDb.fetchAll(); + for aoRow in aaoRows: + # Don't loop forever. + if self.getElapsedSecs() >= config.g_kcSecMaxNewTask: + break; + + # Unpack the data and check if we've rejected the testcasevar/group variation already (they repeat). + oTask = SchedQueueData().initFromDbRow(aoRow); + if config.g_kfSrvGlueDebugScheduler: + self.dprint('** Considering: idItem=%s idGenTestCaseArgs=%s idTestGroup=%s Deps=%s last=%s cfg=%s\n' + % ( oTask.idItem, oTask.idGenTestCaseArgs, oTask.idTestGroup, oTask.aidTestGroupPreReqs, + oTask.tsLastScheduled, oTask.tsConfig,)); + + sRejectNm = '%s:%s' % (oTask.idGenTestCaseArgs, oTask.idTestGroup,); + if sRejectNm in dRejected: + self.dprint('Duplicate, already rejected! (%s)' % (sRejectNm,)); + continue; + dRejected[sRejectNm] = 1; + + # Fetch all the test case info (too much, but who cares right now). + oTestEx = TestCaseArgsDataEx().initFromDbWithGenIdEx(self._oDb, oTask.idGenTestCaseArgs, + tsConfigEff = oTask.tsConfig, + tsRsrcEff = oTask.tsConfig); + if config.g_kfSrvGlueDebugScheduler: + self.dprint('TestCase "%s": %s %s' % (oTestEx.oTestCase.sName, oTestEx.oTestCase.sBaseCmd, oTestEx.sArgs,)); + + # This shouldn't happen, but just in case it does... + if oTestEx.oTestCase.fEnabled is not True: + self.dprint('Testcase is not enabled!!'); + continue; + + # Check if the testbox properties matches the test. + if not oTestEx.matchesTestBoxProps(oTestBoxData): + self.dprint('Testbox mismatch!'); + continue; + + # Try schedule it. + if oTask.idTestSetGangLeader is None or oTestEx.cGangMembers <= 1: + dResponse = self._tryAsLeader(oTask, oTestEx, oTestBoxData, tsNow, sBaseUrl); + elif oTask.cMissingGangMembers > 1: + dResponse = self._tryAsGangMember(oTask, oTestEx, oTestBoxData, tsNow, sBaseUrl); + else: + dResponse = None; # Shouldn't happen! + if dResponse is not None: + self.dprint('Found a task! dResponse=%s' % (dResponse,)); + return dResponse; + + # Found no suitable task. + return None; + + @staticmethod + def _pickSchedGroup(oTestBoxDataEx, iWorkItem, dIgnoreSchedGroupIds): + """ + Picks the next scheduling group for the given testbox. + """ + if len(oTestBoxDataEx.aoInSchedGroups) == 1: + oSchedGroup = oTestBoxDataEx.aoInSchedGroups[0].oSchedGroup; + if oSchedGroup.fEnabled \ + and oSchedGroup.idBuildSrc is not None \ + and oSchedGroup.idSchedGroup not in dIgnoreSchedGroupIds: + return (oSchedGroup, 0); + iWorkItem = 0; + + elif oTestBoxDataEx.aoInSchedGroups: + # Construct priority table of currently enabled scheduling groups. + aaoList1 = []; + for oInGroup in oTestBoxDataEx.aoInSchedGroups: + oSchedGroup = oInGroup.oSchedGroup; + if oSchedGroup.fEnabled and oSchedGroup.idBuildSrc is not None: + iSchedPriority = oInGroup.iSchedPriority; + if iSchedPriority > 31: # paranoia + iSchedPriority = 31; + elif iSchedPriority < 0: # paranoia + iSchedPriority = 0; + + for iSchedPriority in xrange(min(iSchedPriority, len(aaoList1))): + aaoList1[iSchedPriority].append(oSchedGroup); + while len(aaoList1) <= iSchedPriority: + aaoList1.append([oSchedGroup,]); + + # Flatten it into a single list, mixing the priorities a little so it doesn't + # take forever before low priority stuff is executed. + aoFlat = []; + iLo = 0; + iHi = len(aaoList1) - 1; + while iHi >= iLo: + aoFlat += aaoList1[iHi]; + if iLo < iHi: + aoFlat += aaoList1[iLo]; + iLo += 1; + iHi -= 1; + + # Pick the next one. + cLeft = len(aoFlat); + while cLeft > 0: + cLeft -= 1; + iWorkItem += 1; + if iWorkItem >= len(aoFlat) or iWorkItem < 0: + iWorkItem = 0; + if aoFlat[iWorkItem].idSchedGroup not in dIgnoreSchedGroupIds: + return (aoFlat[iWorkItem], iWorkItem); + else: + iWorkItem = 0; + + # No active group. + return (None, iWorkItem); + + @staticmethod + def scheduleNewTask(oDb, oTestBoxData, iWorkItem, sBaseUrl, iVerbosity = 0): + # type: (TMDatabaseConnection, TestBoxData, int, str, int) -> None + """ + Schedules a new task for a testbox. + """ + oTBStatusLogic = TestBoxStatusLogic(oDb); + + try: + # + # To avoid concurrency issues in SchedQueues we lock all the rows + # related to our scheduling queue. Also, since this is a very + # expensive operation we lock the testbox status row to fend of + # repeated retires by faulty testbox scripts. + # + tsSecStart = utils.timestampSecond(); + oDb.rollback(); + oDb.begin(); + oDb.execute('SELECT idTestBox FROM TestBoxStatuses WHERE idTestBox = %s FOR UPDATE NOWAIT' + % (oTestBoxData.idTestBox,)); + oDb.execute('SELECT SchedQueues.idSchedGroup\n' + ' FROM SchedQueues, TestBoxesInSchedGroups\n' + 'WHERE TestBoxesInSchedGroups.idTestBox = %s\n' + ' AND TestBoxesInSchedGroups.tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND TestBoxesInSchedGroups.idSchedGroup = SchedQueues.idSchedGroup\n' + ' FOR UPDATE' + % (oTestBoxData.idTestBox,)); + + # We need the current timestamp. + tsNow = oDb.getCurrentTimestamp(); + + # Re-read the testbox data with scheduling group relations. + oTestBoxDataEx = TestBoxDataEx().initFromDbWithId(oDb, oTestBoxData.idTestBox, tsNow); + if oTestBoxDataEx.fEnabled \ + and oTestBoxDataEx.idGenTestBox == oTestBoxData.idGenTestBox: + + # We may have to skip scheduling groups that are out of work (e.g. 'No build'). + iInitialWorkItem = iWorkItem; + dIgnoreSchedGroupIds = {}; + while True: + # Now, pick the scheduling group. + (oSchedGroup, iWorkItem) = SchedulerBase._pickSchedGroup(oTestBoxDataEx, iWorkItem, dIgnoreSchedGroupIds); + if oSchedGroup is None: + break; + assert oSchedGroup.fEnabled and oSchedGroup.idBuildSrc is not None; + + # Instantiate the specified scheduler and let it do the rest. + oScheduler = SchedulerBase._instantiate(oDb, oSchedGroup, iVerbosity, tsSecStart); + dResponse = oScheduler.scheduleNewTaskWorker(oTestBoxDataEx, tsNow, sBaseUrl); + if dResponse is not None: + oTBStatusLogic.updateWorkItem(oTestBoxDataEx.idTestBox, iWorkItem); + oDb.commit(); + return dResponse; + + # Check out the next work item? + if oScheduler.getElapsedSecs() > config.g_kcSecMaxNewTask: + break; + dIgnoreSchedGroupIds[oSchedGroup.idSchedGroup] = oSchedGroup; + + # No luck, but best if we update the work item if we've made progress. + # Note! In case of a config.g_kcSecMaxNewTask timeout, this may accidentally skip + # a work item with actually work to do. But that's a small price to pay. + if iWorkItem != iInitialWorkItem: + oTBStatusLogic.updateWorkItem(oTestBoxDataEx.idTestBox, iWorkItem); + oDb.commit(); + return None; + except: + oDb.rollback(); + raise; + + # Not enabled, rollback and return no task. + oDb.rollback(); + return None; + + @staticmethod + def tryCancelGangGathering(oDb, oStatusData): + """ + Try canceling a gang gathering. + + Returns True if successfully cancelled. + Returns False if not (someone raced us to the SchedQueue table). + + Note! oStatusData is re-initialized. + """ + assert oStatusData.enmState == TestBoxStatusData.ksTestBoxState_GangGathering; + try: + # + # Lock the tables we're updating so we don't run into concurrency + # issues (we're racing both scheduleNewTask and other callers of + # this method). + # + oDb.rollback(); + oDb.begin(); + oDb.execute('LOCK TABLE TestBoxStatuses, SchedQueues IN EXCLUSIVE MODE'); + + # + # Re-read the testbox data and check that we're still in the same state. + # + oStatusData.initFromDbWithId(oDb, oStatusData.idTestBox); + if oStatusData.enmState == TestBoxStatusData.ksTestBoxState_GangGathering: + # + # Get the leader thru the test set and change the state of the whole gang. + # + oTestSetData = TestSetData().initFromDbWithId(oDb, oStatusData.idTestSet); + + oTBStatusLogic = TestBoxStatusLogic(oDb); + oTBStatusLogic.updateGangStatus(oTestSetData.idTestSetGangLeader, + TestBoxStatusData.ksTestBoxState_GangGatheringTimedOut, + fCommit = False); + + # + # Move the scheduling queue item to the end. + # + oDb.execute('SELECT *\n' + 'FROM SchedQueues\n' + 'WHERE idTestSetGangLeader = %s\n' + , (oTestSetData.idTestSetGangLeader,) ); + oTask = SchedQueueData().initFromDbRow(oDb.fetchOne()); + oTestEx = TestCaseArgsDataEx().initFromDbWithGenIdEx(oDb, oTask.idGenTestCaseArgs, + tsConfigEff = oTask.tsConfig, + tsRsrcEff = oTask.tsConfig); + oDb.execute('UPDATE SchedQueues\n' + ' SET idItem = NEXTVAL(\'SchedQueueItemIdSeq\'),\n' + ' idTestSetGangLeader = NULL,\n' + ' cMissingGangMembers = %s\n' + 'WHERE idItem = %s\n' + , (oTestEx.cGangMembers, oTask.idItem,) ); + + oDb.commit(); + return True; + + if oStatusData.enmState == TestBoxStatusData.ksTestBoxState_GangGatheringTimedOut: + oDb.rollback(); + return True; + except: + oDb.rollback(); + raise; + + # Not enabled, rollback and return no task. + oDb.rollback(); + return False; + + +# +# Unit testing. +# + +# pylint: disable=missing-docstring +class SchedQueueDataTestCase(ModelDataBaseTestCase): + def setUp(self): + self.aoSamples = [SchedQueueData(),]; + +if __name__ == '__main__': + unittest.main(); + # not reached. + diff --git a/src/VBox/ValidationKit/testmanager/core/schedulerbeci.py b/src/VBox/ValidationKit/testmanager/core/schedulerbeci.py new file mode 100755 index 00000000..5cd800d5 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/schedulerbeci.py @@ -0,0 +1,128 @@ +# -*- coding: utf-8 -*- +# $Id: schedulerbeci.py $ + +""" +Test Manager - Best-Effort-Continuous-Integration (BECI) scheduler. +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Validation Kit imports. +from testmanager.core.schedulerbase import SchedulerBase, SchedQueueData; + + +class SchdulerBeci(SchedulerBase): # pylint: disable=too-few-public-methods + """ + The best-effort-continuous-integration scheduler, BECI for short. + """ + + def __init__(self, oDb, oSchedGrpData, iVerbosity, tsSecStart): + SchedulerBase.__init__(self, oDb, oSchedGrpData, iVerbosity, tsSecStart); + + def _recreateQueueItems(self, oData): + # + # Prepare the input data for the loop below. We compress the priority + # to reduce the number of loops we need to executes below. + # + # Note! For BECI test group priority only applies to the ordering of + # test groups, which has been resolved by the done sorting in the + # base class. + # + iMinPriority = 0x7fff; + iMaxPriority = 0; + for oTestGroup in oData.aoTestGroups: + for oTestCase in oTestGroup.aoTestCases: + iPrio = oTestCase.iSchedPriority; + assert iPrio in range(32); + iPrio = iPrio // 4; + assert iPrio in range(8); + if iPrio > iMaxPriority: + iMaxPriority = iPrio; + if iPrio < iMinPriority: + iMinPriority = iPrio; + + oTestCase.iBeciPrio = iPrio; + oTestCase.iNextVariation = -1; + + assert iMinPriority in range(8); + assert iMaxPriority in range(8); + assert iMinPriority <= iMaxPriority; + + # + # Generate the + # + cMaxItems = len(oData.aoArgsVariations) * 64; + cMaxItems = min(cMaxItems, 1048576); + + aoItems = []; + cNotAtEnd = len(oData.aoTestCases); + while len(aoItems) < cMaxItems: + self.msgDebug('outer loop: %s items' % (len(aoItems),)); + for iPrio in range(iMaxPriority, iMinPriority - 1, -1): + #self.msgDebug('prio loop: %s' % (iPrio,)); + for oTestGroup in oData.aoTestGroups: + #self.msgDebug('testgroup loop: %s' % (oTestGroup,)); + for oTestCase in oTestGroup.aoTestCases: + #self.msgDebug('testcase loop: idTestCase=%s' % (oTestCase.idTestCase,)); + if iPrio <= oTestCase.iBeciPrio and oTestCase.aoArgsVariations: + # Get variation. + iNext = oTestCase.iNextVariation; + if iNext != 0: + if iNext == -1: iNext = 0; + cNotAtEnd -= 1; + oArgsVariation = oTestCase.aoArgsVariations[iNext]; + + # Update next variation. + iNext = (iNext + 1) % len(oTestCase.aoArgsVariations); + cNotAtEnd += iNext != 0; + oTestCase.iNextVariation = iNext; + + # Create queue item and append it. + oItem = SchedQueueData(); + oItem.initFromValues(idSchedGroup = self._oSchedGrpData.idSchedGroup, + idGenTestCaseArgs = oArgsVariation.idGenTestCaseArgs, + idTestGroup = oTestGroup.idTestGroup, + aidTestGroupPreReqs = oTestGroup.aidTestGroupPreReqs, + bmHourlySchedule = oTestGroup.bmHourlySchedule, + cMissingGangMembers = oArgsVariation.cGangMembers, + offQueue = len(aoItems)); + aoItems.append(oItem); + + # Done? + if cNotAtEnd == 0: + self.msgDebug('returns %s items' % (len(aoItems),)); + return aoItems; + return aoItems; + diff --git a/src/VBox/ValidationKit/testmanager/core/systemchangelog.py b/src/VBox/ValidationKit/testmanager/core/systemchangelog.py new file mode 100755 index 00000000..7c85c76a --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/systemchangelog.py @@ -0,0 +1,202 @@ +# -*- coding: utf-8 -*- +# $Id: systemchangelog.py $ + +""" +Test Manager - System changelog compilation. +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Validation Kit imports. +from testmanager.core.base import ModelLogicBase; +from testmanager.core.useraccount import UserAccountLogic; +from testmanager.core.systemlog import SystemLogData; + + +class SystemChangelogEntry(object): # pylint: disable=too-many-instance-attributes + """ + System changelog entry. + """ + + def __init__(self, tsEffective, oAuthor, sEvent, idWhat, sDesc): + self.tsEffective = tsEffective; + self.oAuthor = oAuthor; + self.sEvent = sEvent; + self.idWhat = idWhat; + self.sDesc = sDesc; + + +class SystemChangelogLogic(ModelLogicBase): + """ + System changelog compilation logic. + """ + + ## @name What kind of change. + ## @{ + ksWhat_TestBox = 'chlog::TestBox'; + ksWhat_TestCase = 'chlog::TestCase'; + ksWhat_Blacklisting = 'chlog::Blacklisting'; + ksWhat_Build = 'chlog::Build'; + ksWhat_BuildSource = 'chlog::BuildSource'; + ksWhat_FailureCategory = 'chlog::FailureCategory'; + ksWhat_FailureReason = 'chlog::FailureReason'; + ksWhat_GlobalRsrc = 'chlog::GlobalRsrc'; + ksWhat_SchedGroup = 'chlog::SchedGroup'; + ksWhat_TestGroup = 'chlog::TestGroup'; + ksWhat_User = 'chlog::User'; + ksWhat_TestResult = 'chlog::TestResult'; + ## @} + + ## Mapping a changelog entry kind to a table, key and clue. + kdWhatToTable = dict({ # pylint: disable=star-args + ksWhat_TestBox: ( 'TestBoxes', 'idTestBox', None, ), + ksWhat_TestCase: ( 'TestCasees', 'idTestCase', None, ), + ksWhat_Blacklisting: ( 'Blacklist', 'idBlacklisting', None, ), + ksWhat_Build: ( 'Builds', 'idBuild', None, ), + ksWhat_BuildSource: ( 'BuildSources', 'idBuildSrc', None, ), + ksWhat_FailureCategory: ( 'FailureCategories', 'idFailureCategory', None, ), + ksWhat_FailureReason: ( 'FailureReasons', 'idFailureReason', None, ), + ksWhat_GlobalRsrc: ( 'GlobalResources', 'idGlobalRsrc', None, ), + ksWhat_SchedGroup: ( 'SchedGroups', 'idSchedGroup', None, ), + ksWhat_TestGroup: ( 'TestGroups', 'idTestGroup', None, ), + ksWhat_User: ( 'Users', 'idUser', None, ), + ksWhat_TestResult: ( 'TestResults', 'idTestResult', None, ), + }, **{sEvent: ( 'SystemLog', 'tsCreated', 'TimestampId', ) for sEvent in SystemLogData.kasEvents}); + + ## The table key is the effective timestamp. (Can't be used above for some weird scoping reason.) + ksClue_TimestampId = 'TimestampId'; + + ## @todo move to config.py? + ksVSheriffLoginName = 'vsheriff'; + + + ## @name for kaasChangelogTables + ## @internal + ## @{ + ksTweak_None = ''; + ksTweak_NotNullAuthor = 'uidAuthorNotNull'; + ksTweak_NotNullAuthorOrVSheriff = 'uidAuthorNotNullOrVSheriff'; + ## @} + + ## @internal + kaasChangelogTables = ( + # [0]: change name, [1]: Table name, [2]: key column, [3]:later, [4]: tweak + ( ksWhat_TestBox, 'TestBoxes', 'idTestBox', None, ksTweak_NotNullAuthor, ), + ( ksWhat_TestBox, 'TestBoxesInSchedGroups', 'idTestBox', None, ksTweak_None, ), + ( ksWhat_TestCase, 'TestCases', 'idTestCase', None, ksTweak_None, ), + ( ksWhat_TestCase, 'TestCaseArgs', 'idTestCase', None, ksTweak_None, ), + ( ksWhat_TestCase, 'TestCaseDeps', 'idTestCase', None, ksTweak_None, ), + ( ksWhat_TestCase, 'TestCaseGlobalRsrcDeps', 'idTestCase', None, ksTweak_None, ), + ( ksWhat_Blacklisting, 'BuildBlacklist', 'idBlacklisting', None, ksTweak_None, ), + ( ksWhat_Build, 'Builds', 'idBuild', None, ksTweak_NotNullAuthor, ), + ( ksWhat_BuildSource, 'BuildSources', 'idBuildSrc', None, ksTweak_None, ), + ( ksWhat_FailureCategory, 'FailureCategories', 'idFailureCategory', None, ksTweak_None, ), + ( ksWhat_FailureReason, 'FailureReasons', 'idFailureReason', None, ksTweak_None, ), + ( ksWhat_GlobalRsrc, 'GlobalResources', 'idGlobalRsrc', None, ksTweak_None, ), + ( ksWhat_SchedGroup, 'SchedGroups', 'idSchedGroup', None, ksTweak_None, ), + ( ksWhat_SchedGroup, 'SchedGroupMembers', 'idSchedGroup', None, ksTweak_None, ), + ( ksWhat_TestGroup, 'TestGroups', 'idTestGroup', None, ksTweak_None, ), + ( ksWhat_TestGroup, 'TestGroupMembers', 'idTestGroup', None, ksTweak_None, ), + ( ksWhat_User, 'Users', 'uid', None, ksTweak_None, ), + ( ksWhat_TestResult, 'TestResultFailures', 'idTestResult', None, ksTweak_NotNullAuthorOrVSheriff, ), + ); + + def __init__(self, oDb): + ModelLogicBase.__init__(self, oDb); + + + def fetchForListingEx(self, iStart, cMaxRows, tsNow, cDaysBack, aiSortColumns = None): + """ + Fetches SystemLog entries. + + Returns an array (list) of SystemLogData items, empty list if none. + Raises exception on error. + """ + _ = aiSortColumns; + + # + # Construct the query. + # + oUserAccountLogic = UserAccountLogic(self._oDb); + oVSheriff = oUserAccountLogic.tryFetchAccountByLoginName(self.ksVSheriffLoginName); + uidVSheriff = oVSheriff.uid if oVSheriff is not None else -1; + + if tsNow is None: + sWhereTime = self._oDb.formatBindArgs(' WHERE tsEffective >= CURRENT_TIMESTAMP - \'%s days\'::interval\n', + (cDaysBack,)); + else: + sWhereTime = self._oDb.formatBindArgs(' WHERE tsEffective >= (%s::timestamptz - \'%s days\'::interval)\n' + ' AND tsEffective <= %s\n', + (tsNow, cDaysBack, tsNow)); + + # Special entry for the system log. + sQuery = '(\n' + sQuery += ' SELECT NULL AS uidAuthor,\n'; + sQuery += ' tsCreated AS tsEffective,\n'; + sQuery += ' sEvent AS sEvent,\n'; + sQuery += ' NULL AS idWhat,\n'; + sQuery += ' sLogText AS sDesc\n'; + sQuery += ' FROM SystemLog\n'; + sQuery += sWhereTime.replace('tsEffective', 'tsCreated'); + sQuery += ' ORDER BY tsCreated DESC\n' + sQuery += ')' + + for asEntry in self.kaasChangelogTables: + sQuery += ' UNION (\n' + sQuery += ' SELECT uidAuthor, tsEffective, \'' + asEntry[0] + '\', ' + asEntry[2] + ', \'\'\n'; + sQuery += ' FROM ' + asEntry[1] + '\n' + sQuery += sWhereTime; + if asEntry[4] == self.ksTweak_NotNullAuthor or asEntry[4] == self.ksTweak_NotNullAuthorOrVSheriff: + sQuery += ' AND uidAuthor IS NOT NULL\n'; + if asEntry[4] == self.ksTweak_NotNullAuthorOrVSheriff: + sQuery += ' AND uidAuthor <> %u\n' % (uidVSheriff,); + sQuery += ' ORDER BY tsEffective DESC\n' + sQuery += ')'; + sQuery += ' ORDER BY 2 DESC\n'; + sQuery += ' LIMIT %u OFFSET %u\n' % (cMaxRows, iStart, ); + + + # + # Execute the query and construct the return data. + # + self._oDb.execute(sQuery); + aoRows = []; + for aoRow in self._oDb.fetchAll(): + aoRows.append(SystemChangelogEntry(aoRow[1], oUserAccountLogic.cachedLookup(aoRow[0]), + aoRow[2], aoRow[3], aoRow[4])); + + + return aoRows; + diff --git a/src/VBox/ValidationKit/testmanager/core/systemlog.py b/src/VBox/ValidationKit/testmanager/core/systemlog.py new file mode 100755 index 00000000..85929c51 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/systemlog.py @@ -0,0 +1,186 @@ +# -*- coding: utf-8 -*- +# $Id: systemlog.py $ + +""" +Test Manager - SystemLog. +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Standard python imports. +import unittest; + +# Validation Kit imports. +from testmanager.core.base import ModelDataBase, ModelDataBaseTestCase, ModelLogicBase, TMExceptionBase; + + +class SystemLogData(ModelDataBase): # pylint: disable=too-many-instance-attributes + """ + SystemLog Data. + """ + + ## @name Event Constants + # @{ + ksEvent_CmdNacked = 'CmdNack '; + ksEvent_TestBoxUnknown = 'TBoxUnkn'; + ksEvent_TestSetAbandoned = 'TSetAbdd'; + ksEvent_UserAccountUnknown = 'TAccUnkn'; + ksEvent_XmlResultMalformed = 'XmlRMalf'; + ksEvent_SchedQueueRecreate = 'SchQRecr'; + ## @} + + ## Valid event types. + kasEvents = \ + [ \ + ksEvent_CmdNacked, + ksEvent_TestBoxUnknown, + ksEvent_TestSetAbandoned, + ksEvent_UserAccountUnknown, + ksEvent_XmlResultMalformed, + ksEvent_SchedQueueRecreate, + ]; + + ksParam_tsCreated = 'tsCreated'; + ksParam_sEvent = 'sEvent'; + ksParam_sLogText = 'sLogText'; + + kasValidValues_sEvent = kasEvents; + + def __init__(self): + ModelDataBase.__init__(self); + + # + # Initialize with defaults. + # See the database for explanations of each of these fields. + # + self.tsCreated = None; + self.sEvent = None; + self.sLogText = None; + + def initFromDbRow(self, aoRow): + """ + Internal worker for initFromDbWithId and initFromDbWithGenId as well as + SystemLogLogic. + """ + + if aoRow is None: + raise TMExceptionBase('SystemLog row not found.'); + + self.tsCreated = aoRow[0]; + self.sEvent = aoRow[1]; + self.sLogText = aoRow[2]; + return self; + + +class SystemLogLogic(ModelLogicBase): + """ + SystemLog logic. + """ + + def __init__(self, oDb): + ModelLogicBase.__init__(self, oDb); + + def fetchForListing(self, iStart, cMaxRows, tsNow, aiSortColumns = None): + """ + Fetches SystemLog entries. + + Returns an array (list) of SystemLogData items, empty list if none. + Raises exception on error. + """ + _ = aiSortColumns; + if tsNow is None: + self._oDb.execute('SELECT *\n' + 'FROM SystemLog\n' + 'ORDER BY tsCreated DESC\n' + 'LIMIT %s OFFSET %s\n', + (cMaxRows, iStart)); + else: + self._oDb.execute('SELECT *\n' + 'FROM SystemLog\n' + 'WHERE tsCreated <= %s\n' + 'ORDER BY tsCreated DESC\n' + 'LIMIT %s OFFSET %s\n', + (tsNow, cMaxRows, iStart)); + aoRows = []; + for _ in range(self._oDb.getRowCount()): + oData = SystemLogData(); + oData.initFromDbRow(self._oDb.fetchOne()); + aoRows.append(oData); + return aoRows; + + def addEntry(self, sEvent, sLogText, cHoursRepeat = 0, fCommit = False): + """ + Adds an entry to the SystemLog table. + Raises exception on problem. + """ + if sEvent not in SystemLogData.kasEvents: + raise TMExceptionBase('Unknown event type "%s"' % (sEvent,)); + + # Check the repeat restriction first. + if cHoursRepeat > 0: + self._oDb.execute('SELECT COUNT(*) as Stuff\n' + 'FROM SystemLog\n' + 'WHERE tsCreated >= (current_timestamp - interval \'%s hours\')\n' + ' AND sEvent = %s\n' + ' AND sLogText = %s\n', + (cHoursRepeat, + sEvent, + sLogText)); + aRow = self._oDb.fetchOne(); + if aRow[0] > 0: + return None; + + # Insert it. + self._oDb.execute('INSERT INTO SystemLog (sEvent, sLogText)\n' + 'VALUES (%s, %s)\n', + (sEvent, sLogText)); + + if fCommit: + self._oDb.commit(); + return True; + +# +# Unit testing. +# + +# pylint: disable=missing-docstring +class SystemLogDataTestCase(ModelDataBaseTestCase): + def setUp(self): + self.aoSamples = [SystemLogData(),]; + +if __name__ == '__main__': + unittest.main(); + # not reached. + diff --git a/src/VBox/ValidationKit/testmanager/core/testbox.pgsql b/src/VBox/ValidationKit/testmanager/core/testbox.pgsql new file mode 100644 index 00000000..a6a085b8 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/testbox.pgsql @@ -0,0 +1,635 @@ +-- $Id: testbox.pgsql $ +--- @file +-- VBox Test Manager Database Stored Procedures - TestBoxes. +-- + +-- +-- Copyright (C) 2012-2023 Oracle and/or its affiliates. +-- +-- This file is part of VirtualBox base platform packages, as +-- available from https://www.virtualbox.org. +-- +-- This program is free software; you can redistribute it and/or +-- modify it under the terms of the GNU General Public License +-- as published by the Free Software Foundation, in version 3 of the +-- License. +-- +-- This program is distributed in the hope that it will be useful, but +-- WITHOUT ANY WARRANTY; without even the implied warranty of +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +-- General Public License for more details. +-- +-- You should have received a copy of the GNU General Public License +-- along with this program; if not, see <https://www.gnu.org/licenses>. +-- +-- The contents of this file may alternatively be used under the terms +-- of the Common Development and Distribution License Version 1.0 +-- (CDDL), a copy of it is provided in the "COPYING.CDDL" file included +-- in the VirtualBox distribution, in which case the provisions of the +-- CDDL are applicable instead of those of the GPL. +-- +-- You may elect to license modified versions of this file under the +-- terms and conditions of either the GPL or the CDDL or both. +-- +-- SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +-- + + +-- +-- Old type signatures. +-- +DROP FUNCTION IF EXISTS TestBoxLogic_addEntry(a_uidAuthor INTEGER, + a_ip inet, + a_uuidSystem uuid, + a_sName TEXT, + a_sDescription TEXT, + a_idSchedGroup INTEGER, + a_fEnabled BOOLEAN, + a_enmLomKind LomKind_T, + a_ipLom inet, + a_pctScaleTimeout INTEGER, -- Actually smallint, but default typing fun. + a_sComment TEXT, + a_enmPendingCmd TestBoxCmd_T, + OUT r_idTestBox INTEGER, + OUT r_idGenTestBox INTEGER, + OUT r_tsEffective TIMESTAMP WITH TIME ZONE); +DROP FUNCTION IF EXISTS TestBoxLogic_editEntry(a_uidAuthor INTEGER, + a_idTestBox INTEGER, + a_ip inet, + a_uuidSystem uuid, + a_sName TEXT, + a_sDescription TEXT, + a_idSchedGroup INTEGER, + a_fEnabled BOOLEAN, + a_enmLomKind LomKind_T, + a_ipLom inet, + a_pctScaleTimeout INTEGER, -- Actually smallint, but default typing fun. + a_sComment TEXT, + a_enmPendingCmd TestBoxCmd_T, + OUT r_idGenTestBox INTEGER, + OUT r_tsEffective TIMESTAMP WITH TIME ZONE); +DROP FUNCTION IF EXISTS TestBoxLogic_removeEntry(INTEGER, INTEGER, BOOLEAN); +DROP FUNCTION IF EXISTS TestBoxLogic_addGroupEntry(a_uidAuthor INTEGER, + a_idTestBox INTEGER, + a_idSchedGroup INTEGER, + a_iSchedPriority INTEGER, + OUT r_tsEffective TIMESTAMP WITH TIME ZONE); +DROP FUNCTION IF EXISTS TestBoxLogic_editGroupEntry(a_uidAuthor INTEGER, + a_idTestBox INTEGER, + a_idSchedGroup INTEGER, + a_iSchedPriority INTEGER, + OUT r_tsEffective INTEGER); + + +--- +-- Checks if the test box name is unique, ignoring a_idTestCaseIgnore. +-- Raises exception if duplicates are found. +-- +-- @internal +-- +CREATE OR REPLACE FUNCTION TestBoxLogic_checkUniqueName(a_sName TEXT, a_idTestBoxIgnore INTEGER) + RETURNS VOID AS $$ + DECLARE + v_cRows INTEGER; + BEGIN + SELECT COUNT(*) INTO v_cRows + FROM TestBoxes + WHERE sName = a_sName + AND tsExpire = 'infinity'::TIMESTAMP + AND idTestBox <> a_idTestBoxIgnore; + IF v_cRows <> 0 THEN + RAISE EXCEPTION 'Duplicate test box name "%" (% times)', a_sName, v_cRows; + END IF; + END; +$$ LANGUAGE plpgsql; + + +--- +-- Checks that the given scheduling group exists. +-- Raises exception if it doesn't. +-- +-- @internal +-- +CREATE OR REPLACE FUNCTION TestBoxLogic_checkSchedGroupExists(a_idSchedGroup INTEGER) + RETURNS VOID AS $$ + DECLARE + v_cRows INTEGER; + BEGIN + SELECT COUNT(*) INTO v_cRows + FROM SchedGroups + WHERE idSchedGroup = a_idSchedGroup + AND tsExpire = 'infinity'::TIMESTAMP; + IF v_cRows <> 1 THEN + IF v_cRows = 0 THEN + RAISE EXCEPTION 'Scheduling group with ID % was not found', a_idSchedGroup; + END IF; + RAISE EXCEPTION 'Integrity error in SchedGroups: % current rows with idSchedGroup=%', v_cRows, a_idSchedGroup; + END IF; + END; +$$ LANGUAGE plpgsql; + + +--- +-- Checks that the given testbxo + scheduling group pair does not currently exists. +-- Raises exception if it does. +-- +-- @internal +-- +CREATE OR REPLACE FUNCTION TestBoxLogic_checkTestBoxNotInSchedGroup(a_idTestBox INTEGER, a_idSchedGroup INTEGER) + RETURNS VOID AS $$ + DECLARE + v_cRows INTEGER; + BEGIN + SELECT COUNT(*) INTO v_cRows + FROM TestBoxesInSchedGroups + WHERE idTestBox = a_idTestBox + AND idSchedGroup = a_idSchedGroup + AND tsExpire = 'infinity'::TIMESTAMP; + IF v_cRows <> 0 THEN + RAISE EXCEPTION 'TestBox % is already a member of scheduling group %', a_idTestBox, a_idSchedGroup; + END IF; + END; +$$ LANGUAGE plpgsql; + + +--- +-- Historize a row. +-- @internal +-- +CREATE OR REPLACE FUNCTION TestBoxLogic_historizeEntry(a_idGenTestBox INTEGER, a_tsExpire TIMESTAMP WITH TIME ZONE) + RETURNS VOID AS $$ + DECLARE + v_cUpdatedRows INTEGER; + BEGIN + UPDATE TestBoxes + SET tsExpire = a_tsExpire + WHERE idGenTestBox = a_idGenTestBox + AND tsExpire = 'infinity'::TIMESTAMP; + GET DIAGNOSTICS v_cUpdatedRows = ROW_COUNT; + IF v_cUpdatedRows <> 1 THEN + IF v_cUpdatedRows = 0 THEN + RAISE EXCEPTION 'Test box generation ID % is no longer valid', a_idGenTestBox; + END IF; + RAISE EXCEPTION 'Integrity error in TestBoxes: % current rows with idGenTestBox=%', v_cUpdatedRows, a_idGenTestBox; + END IF; + END; +$$ LANGUAGE plpgsql; + + +--- +-- Historize a in-scheduling-group row. +-- @internal +-- +CREATE OR REPLACE FUNCTION TestBoxLogic_historizeGroupEntry(a_idTestBox INTEGER, + a_idSchedGroup INTEGER, + a_tsExpire TIMESTAMP WITH TIME ZONE) + RETURNS VOID AS $$ + DECLARE + v_cUpdatedRows INTEGER; + BEGIN + UPDATE TestBoxesInSchedGroups + SET tsExpire = a_tsExpire + WHERE idTestBox = a_idTestBox + AND idSchedGroup = a_idSchedGroup + AND tsExpire = 'infinity'::TIMESTAMP; + GET DIAGNOSTICS v_cUpdatedRows = ROW_COUNT; + IF v_cUpdatedRows <> 1 THEN + IF v_cUpdatedRows = 0 THEN + RAISE EXCEPTION 'TestBox ID % / SchedGroup ID % is no longer a valid combination', a_idTestBox, a_idSchedGroup; + END IF; + RAISE EXCEPTION 'Integrity error in TestBoxesInSchedGroups: % current rows for % / %', + v_cUpdatedRows, a_idTestBox, a_idSchedGroup; + END IF; + END; +$$ LANGUAGE plpgsql; + + +--- +-- Translate string via the string table. +-- +-- @returns NULL if a_sValue is NULL, otherwise a string ID. +-- +CREATE OR REPLACE FUNCTION TestBoxLogic_lookupOrFindString(a_sValue TEXT) + RETURNS INTEGER AS $$ + DECLARE + v_idStr INTEGER; + v_cRows INTEGER; + BEGIN + IF a_sValue IS NULL THEN + RETURN NULL; + END IF; + + SELECT idStr + INTO v_idStr + FROM TestBoxStrTab + WHERE sValue = a_sValue; + GET DIAGNOSTICS v_cRows = ROW_COUNT; + IF v_cRows = 0 THEN + INSERT INTO TestBoxStrTab (sValue) + VALUES (a_sValue) + RETURNING idStr INTO v_idStr; + END IF; + RETURN v_idStr; + END; +$$ LANGUAGE plpgsql; + + +--- +-- Only adds the user settable parts of the row, i.e. not what TestBoxLogic_updateOnSignOn touches. +-- +CREATE OR REPLACE function TestBoxLogic_addEntry(a_uidAuthor INTEGER, + a_ip inet, + a_uuidSystem uuid, + a_sName TEXT, + a_sDescription TEXT, + a_fEnabled BOOLEAN, + a_enmLomKind LomKind_T, + a_ipLom inet, + a_pctScaleTimeout INTEGER, -- Actually smallint, but default typing fun. + a_sComment TEXT, + a_enmPendingCmd TestBoxCmd_T, + OUT r_idTestBox INTEGER, + OUT r_idGenTestBox INTEGER, + OUT r_tsEffective TIMESTAMP WITH TIME ZONE + ) AS $$ + DECLARE + v_idStrDescription INTEGER; + v_idStrComment INTEGER; + BEGIN + PERFORM TestBoxLogic_checkUniqueName(a_sName, -1); + + SELECT TestBoxLogic_lookupOrFindString(a_sDescription) INTO v_idStrDescription; + SELECT TestBoxLogic_lookupOrFindString(a_sComment) INTO v_idStrComment; + + INSERT INTO TestBoxes ( + tsEffective, -- 1 + uidAuthor, -- 2 + ip, -- 3 + uuidSystem, -- 4 + sName, -- 5 + idStrDescription, -- 6 + fEnabled, -- 7 + enmLomKind, -- 8 + ipLom, -- 9 + pctScaleTimeout, -- 10 + idStrComment, -- 11 + enmPendingCmd ) -- 12 + VALUES (CURRENT_TIMESTAMP, -- 1 + a_uidAuthor, -- 2 + a_ip, -- 3 + a_uuidSystem, -- 4 + a_sName, -- 5 + v_idStrDescription, -- 6 + a_fEnabled, -- 7 + a_enmLomKind, -- 8 + a_ipLom, -- 9 + a_pctScaleTimeout, -- 10 + v_idStrComment, -- 11 + a_enmPendingCmd ) -- 12 + RETURNING idTestBox, idGenTestBox, tsEffective INTO r_idTestBox, r_idGenTestBox, r_tsEffective; + END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE function TestBoxLogic_addGroupEntry(a_uidAuthor INTEGER, + a_idTestBox INTEGER, + a_idSchedGroup INTEGER, + a_iSchedPriority INTEGER, + OUT r_tsEffective TIMESTAMP WITH TIME ZONE + ) AS $$ + BEGIN + PERFORM TestBoxLogic_checkSchedGroupExists(a_idSchedGroup); + PERFORM TestBoxLogic_checkTestBoxNotInSchedGroup(a_idTestBox, a_idSchedGroup); + + INSERT INTO TestBoxesInSchedGroups ( + idTestBox, + idSchedGroup, + tsEffective, + tsExpire, + uidAuthor, + iSchedPriority) + VALUES (a_idTestBox, + a_idSchedGroup, + CURRENT_TIMESTAMP, + 'infinity'::TIMESTAMP, + a_uidAuthor, + a_iSchedPriority) + RETURNING tsEffective INTO r_tsEffective; + END; +$$ LANGUAGE plpgsql; + + +--- +-- Only adds the user settable parts of the row, i.e. not what TestBoxLogic_updateOnSignOn touches. +-- +CREATE OR REPLACE function TestBoxLogic_editEntry(a_uidAuthor INTEGER, + a_idTestBox INTEGER, + a_ip inet, + a_uuidSystem uuid, + a_sName TEXT, + a_sDescription TEXT, + a_fEnabled BOOLEAN, + a_enmLomKind LomKind_T, + a_ipLom inet, + a_pctScaleTimeout INTEGER, -- Actually smallint, but default typing fun. + a_sComment TEXT, + a_enmPendingCmd TestBoxCmd_T, + OUT r_idGenTestBox INTEGER, + OUT r_tsEffective TIMESTAMP WITH TIME ZONE + ) AS $$ + DECLARE + v_Row TestBoxes%ROWTYPE; + v_idStrDescription INTEGER; + v_idStrComment INTEGER; + BEGIN + PERFORM TestBoxLogic_checkUniqueName(a_sName, a_idTestBox); + + SELECT TestBoxLogic_lookupOrFindString(a_sDescription) INTO v_idStrDescription; + SELECT TestBoxLogic_lookupOrFindString(a_sComment) INTO v_idStrComment; + + -- Fetch and historize the current row - there must be one. + UPDATE TestBoxes + SET tsExpire = CURRENT_TIMESTAMP + WHERE idTestBox = a_idTestBox + AND tsExpire = 'infinity'::TIMESTAMP + RETURNING * INTO STRICT v_Row; + + -- Modify the row with the new data. + v_Row.uidAuthor := a_uidAuthor; + v_Row.ip := a_ip; + v_Row.uuidSystem := a_uuidSystem; + v_Row.sName := a_sName; + v_Row.idStrDescription := v_idStrDescription; + v_Row.fEnabled := a_fEnabled; + v_Row.enmLomKind := a_enmLomKind; + v_Row.ipLom := a_ipLom; + v_Row.pctScaleTimeout := a_pctScaleTimeout; + v_Row.idStrComment := v_idStrComment; + v_Row.enmPendingCmd := a_enmPendingCmd; + v_Row.tsEffective := v_Row.tsExpire; + r_tsEffective := v_Row.tsExpire; + v_Row.tsExpire := 'infinity'::TIMESTAMP; + + -- Get a new generation ID. + SELECT NEXTVAL('TestBoxGenIdSeq') INTO v_Row.idGenTestBox; + r_idGenTestBox := v_Row.idGenTestBox; + + -- Insert the modified row. + INSERT INTO TestBoxes VALUES (v_Row.*); + END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE function TestBoxLogic_editGroupEntry(a_uidAuthor INTEGER, + a_idTestBox INTEGER, + a_idSchedGroup INTEGER, + a_iSchedPriority INTEGER, + OUT r_tsEffective TIMESTAMP WITH TIME ZONE + ) AS $$ + DECLARE + v_Row TestBoxesInSchedGroups%ROWTYPE; + v_idStrDescription INTEGER; + v_idStrComment INTEGER; + BEGIN + PERFORM TestBoxLogic_checkSchedGroupExists(a_idSchedGroup); + + -- Fetch and historize the current row - there must be one. + UPDATE TestBoxesInSchedGroups + SET tsExpire = CURRENT_TIMESTAMP + WHERE idTestBox = a_idTestBox + AND idSchedGroup = a_idSchedGroup + AND tsExpire = 'infinity'::TIMESTAMP + RETURNING * INTO STRICT v_Row; + + -- Modify the row with the new data. + v_Row.uidAuthor := a_uidAuthor; + v_Row.iSchedPriority := a_iSchedPriority; + v_Row.tsEffective := v_Row.tsExpire; + r_tsEffective := v_Row.tsExpire; + v_Row.tsExpire := 'infinity'::TIMESTAMP; + + -- Insert the modified row. + INSERT INTO TestBoxesInSchedGroups VALUES (v_Row.*); + END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION TestBoxLogic_removeEntry(a_uidAuthor INTEGER, a_idTestBox INTEGER, a_fCascade BOOLEAN) + RETURNS VOID AS $$ + DECLARE + v_Row TestBoxes%ROWTYPE; + v_tsEffective TIMESTAMP WITH TIME ZONE; + v_Rec RECORD; + v_sErrors TEXT; + BEGIN + -- + -- Check preconditions. + -- + IF a_fCascade <> TRUE THEN + -- @todo implement checks which throws useful exceptions. + ELSE + RAISE EXCEPTION 'CASCADE test box deletion is not implemented'; + END IF; + + -- + -- Delete all current groups, skipping history since we're also deleting the testbox. + -- + UPDATE TestBoxesInSchedGroups + SET tsExpire = CURRENT_TIMESTAMP + WHERE idTestBox = a_idTestBox + AND tsExpire = 'infinity'::TIMESTAMP; + + -- + -- To preserve the information about who deleted the record, we try to + -- add a dummy record which expires immediately. I say try because of + -- the primary key, we must let the new record be valid for 1 us. :-( + -- + SELECT * INTO STRICT v_Row + FROM TestBoxes + WHERE idTestBox = a_idTestBox + AND tsExpire = 'infinity'::TIMESTAMP; + + v_tsEffective := CURRENT_TIMESTAMP - INTERVAL '1 microsecond'; + IF v_Row.tsEffective < v_tsEffective THEN + PERFORM TestBoxLogic_historizeEntry(v_Row.idGenTestBox, v_tsEffective); + + v_Row.tsEffective := v_tsEffective; + v_Row.tsExpire := CURRENT_TIMESTAMP; + v_Row.uidAuthor := a_uidAuthor; + SELECT NEXTVAL('TestBoxGenIdSeq') INTO v_Row.idGenTestBox; + INSERT INTO TestBoxes VALUES (v_Row.*); + ELSE + PERFORM TestBoxLogic_historizeEntry(v_Row.idGenTestBox, CURRENT_TIMESTAMP); + END IF; + + EXCEPTION + WHEN NO_DATA_FOUND THEN + RAISE EXCEPTION 'Test box with ID % does not currently exist', a_idTestBox; + WHEN TOO_MANY_ROWS THEN + RAISE EXCEPTION 'Integrity error in TestBoxes: Too many current rows for %', a_idTestBox; + END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION TestBoxLogic_removeGroupEntry(a_uidAuthor INTEGER, a_idTestBox INTEGER, a_idSchedGroup INTEGER) + RETURNS VOID AS $$ + DECLARE + v_Row TestBoxesInSchedGroups%ROWTYPE; + v_tsEffective TIMESTAMP WITH TIME ZONE; + BEGIN + -- + -- To preserve the information about who deleted the record, we try to + -- add a dummy record which expires immediately. I say try because of + -- the primary key, we must let the new record be valid for 1 us. :-( + -- + SELECT * INTO STRICT v_Row + FROM TestBoxesInSchedGroups + WHERE idTestBox = a_idTestBox + AND idSchedGroup = a_idSchedGroup + AND tsExpire = 'infinity'::TIMESTAMP; + + v_tsEffective := CURRENT_TIMESTAMP - INTERVAL '1 microsecond'; + IF v_Row.tsEffective < v_tsEffective THEN + PERFORM TestBoxLogic_historizeGroupEntry(a_idTestBox, a_idSchedGroup, v_tsEffective); + + v_Row.tsEffective := v_tsEffective; + v_Row.tsExpire := CURRENT_TIMESTAMP; + v_Row.uidAuthor := a_uidAuthor; + INSERT INTO TestBoxesInSchedGroups VALUES (v_Row.*); + ELSE + PERFORM TestBoxLogic_historizeGroupEntry(a_idTestBox, a_idSchedGroup, CURRENT_TIMESTAMP); + END IF; + + EXCEPTION + WHEN NO_DATA_FOUND THEN + RAISE EXCEPTION 'TestBox #% does is not currently a member of scheduling group #%', a_idTestBox, a_idSchedGroup; + WHEN TOO_MANY_ROWS THEN + RAISE EXCEPTION 'Integrity error in TestBoxesInSchedGroups: Too many current rows for % / %', + a_idTestBox, a_idSchedGroup; + END; +$$ LANGUAGE plpgsql; + + +--- +-- Sign on update +-- +CREATE OR REPLACE function TestBoxLogic_updateOnSignOn(a_idTestBox INTEGER, + a_ip inet, + a_sOs TEXT, + a_sOsVersion TEXT, + a_sCpuVendor TEXT, + a_sCpuArch TEXT, + a_sCpuName TEXT, + a_lCpuRevision bigint, + a_cCpus INTEGER, -- Actually smallint, but default typing fun. + a_fCpuHwVirt boolean, + a_fCpuNestedPaging boolean, + a_fCpu64BitGuest boolean, + a_fChipsetIoMmu boolean, + a_fRawMode boolean, + a_cMbMemory bigint, + a_cMbScratch bigint, + a_sReport TEXT, + a_iTestBoxScriptRev INTEGER, + a_iPythonHexVersion INTEGER, + OUT r_idGenTestBox INTEGER + ) AS $$ + DECLARE + v_Row TestBoxes%ROWTYPE; + v_idStrOs INTEGER; + v_idStrOsVersion INTEGER; + v_idStrCpuVendor INTEGER; + v_idStrCpuArch INTEGER; + v_idStrCpuName INTEGER; + v_idStrReport INTEGER; + BEGIN + SELECT TestBoxLogic_lookupOrFindString(a_sOs) INTO v_idStrOs; + SELECT TestBoxLogic_lookupOrFindString(a_sOsVersion) INTO v_idStrOsVersion; + SELECT TestBoxLogic_lookupOrFindString(a_sCpuVendor) INTO v_idStrCpuVendor; + SELECT TestBoxLogic_lookupOrFindString(a_sCpuArch) INTO v_idStrCpuArch; + SELECT TestBoxLogic_lookupOrFindString(a_sCpuName) INTO v_idStrCpuName; + SELECT TestBoxLogic_lookupOrFindString(a_sReport) INTO v_idStrReport; + + -- Fetch and historize the current row - there must be one. + UPDATE TestBoxes + SET tsExpire = CURRENT_TIMESTAMP + WHERE idTestBox = a_idTestBox + AND tsExpire = 'infinity'::TIMESTAMP + RETURNING * INTO STRICT v_Row; + + -- Modify the row with the new data. + v_Row.uidAuthor := NULL; + v_Row.ip := a_ip; + v_Row.idStrOs := v_idStrOs; + v_Row.idStrOsVersion := v_idStrOsVersion; + v_Row.idStrCpuVendor := v_idStrCpuVendor; + v_Row.idStrCpuArch := v_idStrCpuArch; + v_Row.idStrCpuName := v_idStrCpuName; + v_Row.lCpuRevision := a_lCpuRevision; + v_Row.cCpus := a_cCpus; + v_Row.fCpuHwVirt := a_fCpuHwVirt; + v_Row.fCpuNestedPaging := a_fCpuNestedPaging; + v_Row.fCpu64BitGuest := a_fCpu64BitGuest; + v_Row.fChipsetIoMmu := a_fChipsetIoMmu; + v_Row.fRawMode := a_fRawMode; + v_Row.cMbMemory := a_cMbMemory; + v_Row.cMbScratch := a_cMbScratch; + v_Row.idStrReport := v_idStrReport; + v_Row.iTestBoxScriptRev := a_iTestBoxScriptRev; + v_Row.iPythonHexVersion := a_iPythonHexVersion; + v_Row.tsEffective := v_Row.tsExpire; + v_Row.tsExpire := 'infinity'::TIMESTAMP; + + -- Get a new generation ID. + SELECT NEXTVAL('TestBoxGenIdSeq') INTO v_Row.idGenTestBox; + r_idGenTestBox := v_Row.idGenTestBox; + + -- Insert the modified row. + INSERT INTO TestBoxes VALUES (v_Row.*); + END; +$$ LANGUAGE plpgsql; + + +--- +-- Set new command. +-- +CREATE OR REPLACE function TestBoxLogic_setCommand(a_uidAuthor INTEGER, + a_idTestBox INTEGER, + a_enmOldCmd TestBoxCmd_T, + a_enmNewCmd TestBoxCmd_T, + a_sComment TEXT, + OUT r_idGenTestBox INTEGER, + OUT r_tsEffective TIMESTAMP WITH TIME ZONE + ) AS $$ + DECLARE + v_Row TestBoxes%ROWTYPE; + v_idStrComment INTEGER; + BEGIN + SELECT TestBoxLogic_lookupOrFindString(a_sComment) INTO v_idStrComment; + + -- Fetch and historize the current row - there must be one. + UPDATE TestBoxes + SET tsExpire = CURRENT_TIMESTAMP + WHERE idTestBox = a_idTestBox + AND tsExpire = 'infinity'::TIMESTAMP + AND enmPendingCmd = a_enmOldCmd + RETURNING * INTO STRICT v_Row; + + -- Modify the row with the new data. + v_Row.enmPendingCmd := a_enmNewCmd; + IF v_idStrComment IS NOT NULL THEN + v_Row.idStrComment := v_idStrComment; + END IF; + v_Row.tsEffective := v_Row.tsExpire; + r_tsEffective := v_Row.tsExpire; + v_Row.tsExpire := 'infinity'::TIMESTAMP; + + -- Get a new generation ID. + SELECT NEXTVAL('TestBoxGenIdSeq') INTO v_Row.idGenTestBox; + r_idGenTestBox := v_Row.idGenTestBox; + + -- Insert the modified row. + INSERT INTO TestBoxes VALUES (v_Row.*); + END; +$$ LANGUAGE plpgsql; + diff --git a/src/VBox/ValidationKit/testmanager/core/testbox.py b/src/VBox/ValidationKit/testmanager/core/testbox.py new file mode 100755 index 00000000..6686ca3b --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/testbox.py @@ -0,0 +1,1286 @@ +# -*- coding: utf-8 -*- +# $Id: testbox.py $ + +""" +Test Manager - TestBox. +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Standard python imports. +import copy; +import sys; +import unittest; + +# Validation Kit imports. +from testmanager.core import db; +from testmanager.core.base import ModelDataBase, ModelDataBaseTestCase, ModelLogicBase, TMInFligthCollision, \ + TMInvalidData, TMTooManyRows, TMRowNotFound, \ + ChangeLogEntry, AttributeChangeEntry, AttributeChangeEntryPre; +from testmanager.core.useraccount import UserAccountLogic; + +# Python 3 hacks: +if sys.version_info[0] >= 3: + xrange = range; # pylint: disable=redefined-builtin,invalid-name + + +class TestBoxInSchedGroupData(ModelDataBase): + """ + TestBox in SchedGroup data. + """ + + ksParam_idTestBox = 'TestBoxInSchedGroup_idTestBox'; + ksParam_idSchedGroup = 'TestBoxInSchedGroup_idSchedGroup'; + ksParam_tsEffective = 'TestBoxInSchedGroup_tsEffective'; + ksParam_tsExpire = 'TestBoxInSchedGroup_tsExpire'; + ksParam_uidAuthor = 'TestBoxInSchedGroup_uidAuthor'; + ksParam_iSchedPriority = 'TestBoxInSchedGroup_iSchedPriority'; + + kasAllowNullAttributes = [ 'tsEffective', 'tsExpire', 'uidAuthor', ] + + kiMin_iSchedPriority = 0; + kiMax_iSchedPriority = 32; + + kcDbColumns = 6; + + def __init__(self): + ModelDataBase.__init__(self); + self.idTestBox = None; + self.idSchedGroup = None; + self.tsEffective = None; + self.tsExpire = None; + self.uidAuthor = None; + self.iSchedPriority = 16; + + def initFromDbRow(self, aoRow): + """ + Expecting the result from a query like this: + SELECT * FROM TestBoxesInSchedGroups + """ + if aoRow is None: + raise TMRowNotFound('TestBox/SchedGroup not found.'); + + self.idTestBox = aoRow[0]; + self.idSchedGroup = aoRow[1]; + self.tsEffective = aoRow[2]; + self.tsExpire = aoRow[3]; + self.uidAuthor = aoRow[4]; + self.iSchedPriority = aoRow[5]; + + return self; + +class TestBoxInSchedGroupDataEx(TestBoxInSchedGroupData): + """ + Extended version of TestBoxInSchedGroupData that contains the scheduling group. + """ + + def __init__(self): + TestBoxInSchedGroupData.__init__(self); + self.oSchedGroup = None # type: SchedGroupData + + def initFromDbRowEx(self, aoRow, oDb, tsNow = None, sPeriodBack = None): + """ + Extended version of initFromDbRow that fills in the rest from the database. + """ + from testmanager.core.schedgroup import SchedGroupData; + self.initFromDbRow(aoRow); + self.oSchedGroup = SchedGroupData().initFromDbWithId(oDb, self.idSchedGroup, tsNow, sPeriodBack); + return self; + +class TestBoxDataForSchedGroup(TestBoxInSchedGroupData): + """ + Extended version of TestBoxInSchedGroupData that adds the testbox data (if available). + Used by TestBoxLogic.fetchForSchedGroup + """ + + def __init__(self): + TestBoxInSchedGroupData.__init__(self); + self.oTestBox = None # type: TestBoxData + + def initFromDbRow(self, aoRow): + """ + The row is: TestBoxesInSchedGroups.*, TestBoxesWithStrings.* + """ + TestBoxInSchedGroupData.initFromDbRow(self, aoRow); + if aoRow[self.kcDbColumns]: + self.oTestBox = TestBoxData().initFromDbRow(aoRow[self.kcDbColumns:]); + else: + self.oTestBox = None; + return self; + + def getDataAttributes(self): + asAttributes = TestBoxInSchedGroupData.getDataAttributes(self); + asAttributes.remove('oTestBox'); + return asAttributes; + + def _validateAndConvertWorker(self, asAllowNullAttributes, oDb, enmValidateFor = ModelDataBase.ksValidateFor_Other): + dErrors = TestBoxInSchedGroupData._validateAndConvertWorker(self, asAllowNullAttributes, oDb, enmValidateFor); + if self.ksParam_idTestBox not in dErrors: + self.oTestBox = TestBoxData(); + try: + self.oTestBox.initFromDbWithId(oDb, self.idTestBox); + except Exception as oXcpt: + self.oTestBox = TestBoxData() + dErrors[self.ksParam_idTestBox] = str(oXcpt); + return dErrors; + + +# pylint: disable=invalid-name +class TestBoxData(ModelDataBase): # pylint: disable=too-many-instance-attributes + """ + TestBox Data. + """ + + ## LomKind_T + ksLomKind_None = 'none'; + ksLomKind_ILOM = 'ilom'; + ksLomKind_ELOM = 'elom'; + ksLomKind_AppleXserveLom = 'apple-xserver-lom'; + kasLomKindValues = [ ksLomKind_None, ksLomKind_ILOM, ksLomKind_ELOM, ksLomKind_AppleXserveLom]; + kaoLomKindDescs = \ + [ + ( ksLomKind_None, 'None', ''), + ( ksLomKind_ILOM, 'ILOM', ''), + ( ksLomKind_ELOM, 'ELOM', ''), + ( ksLomKind_AppleXserveLom, 'Apple Xserve LOM', ''), + ]; + + + ## TestBoxCmd_T + ksTestBoxCmd_None = 'none'; + ksTestBoxCmd_Abort = 'abort'; + ksTestBoxCmd_Reboot = 'reboot'; + ksTestBoxCmd_Upgrade = 'upgrade'; + ksTestBoxCmd_UpgradeAndReboot = 'upgrade-and-reboot'; + ksTestBoxCmd_Special = 'special'; + kasTestBoxCmdValues = [ ksTestBoxCmd_None, ksTestBoxCmd_Abort, ksTestBoxCmd_Reboot, ksTestBoxCmd_Upgrade, + ksTestBoxCmd_UpgradeAndReboot, ksTestBoxCmd_Special]; + kaoTestBoxCmdDescs = \ + [ + ( ksTestBoxCmd_None, 'None', ''), + ( ksTestBoxCmd_Abort, 'Abort current test', ''), + ( ksTestBoxCmd_Reboot, 'Reboot TestBox', ''), + ( ksTestBoxCmd_Upgrade, 'Upgrade TestBox Script', ''), + ( ksTestBoxCmd_UpgradeAndReboot, 'Upgrade TestBox Script and reboot', ''), + ( ksTestBoxCmd_Special, 'Special (reserved)', ''), + ]; + + + ksIdAttr = 'idTestBox'; + ksIdGenAttr = 'idGenTestBox'; + + ksParam_idTestBox = 'TestBox_idTestBox'; + ksParam_tsEffective = 'TestBox_tsEffective'; + ksParam_tsExpire = 'TestBox_tsExpire'; + ksParam_uidAuthor = 'TestBox_uidAuthor'; + ksParam_idGenTestBox = 'TestBox_idGenTestBox'; + ksParam_ip = 'TestBox_ip'; + ksParam_uuidSystem = 'TestBox_uuidSystem'; + ksParam_sName = 'TestBox_sName'; + ksParam_sDescription = 'TestBox_sDescription'; + ksParam_fEnabled = 'TestBox_fEnabled'; + ksParam_enmLomKind = 'TestBox_enmLomKind'; + ksParam_ipLom = 'TestBox_ipLom'; + ksParam_pctScaleTimeout = 'TestBox_pctScaleTimeout'; + ksParam_sComment = 'TestBox_sComment'; + ksParam_sOs = 'TestBox_sOs'; + ksParam_sOsVersion = 'TestBox_sOsVersion'; + ksParam_sCpuVendor = 'TestBox_sCpuVendor'; + ksParam_sCpuArch = 'TestBox_sCpuArch'; + ksParam_sCpuName = 'TestBox_sCpuName'; + ksParam_lCpuRevision = 'TestBox_lCpuRevision'; + ksParam_cCpus = 'TestBox_cCpus'; + ksParam_fCpuHwVirt = 'TestBox_fCpuHwVirt'; + ksParam_fCpuNestedPaging = 'TestBox_fCpuNestedPaging'; + ksParam_fCpu64BitGuest = 'TestBox_fCpu64BitGuest'; + ksParam_fChipsetIoMmu = 'TestBox_fChipsetIoMmu'; + ksParam_fRawMode = 'TestBox_fRawMode'; + ksParam_cMbMemory = 'TestBox_cMbMemory'; + ksParam_cMbScratch = 'TestBox_cMbScratch'; + ksParam_sReport = 'TestBox_sReport'; + ksParam_iTestBoxScriptRev = 'TestBox_iTestBoxScriptRev'; + ksParam_iPythonHexVersion = 'TestBox_iPythonHexVersion'; + ksParam_enmPendingCmd = 'TestBox_enmPendingCmd'; + + kasInternalAttributes = [ 'idStrDescription', 'idStrComment', 'idStrOs', 'idStrOsVersion', 'idStrCpuVendor', + 'idStrCpuArch', 'idStrCpuName', 'idStrReport', ]; + kasMachineSettableOnly = [ 'sOs', 'sOsVersion', 'sCpuVendor', 'sCpuArch', 'sCpuName', 'lCpuRevision', 'cCpus', + 'fCpuHwVirt', 'fCpuNestedPaging', 'fCpu64BitGuest', 'fChipsetIoMmu', 'fRawMode', + 'cMbMemory', 'cMbScratch', 'sReport', 'iTestBoxScriptRev', 'iPythonHexVersion', ]; + kasAllowNullAttributes = ['idTestBox', 'tsEffective', 'tsExpire', 'uidAuthor', 'idGenTestBox', 'sDescription', + 'ipLom', 'sComment', ] + kasMachineSettableOnly + kasInternalAttributes; + + kasValidValues_enmLomKind = kasLomKindValues; + kasValidValues_enmPendingCmd = kasTestBoxCmdValues; + kiMin_pctScaleTimeout = 11; + kiMax_pctScaleTimeout = 19999; + kcchMax_sReport = 65535; + + kcDbColumns = 40; # including the 7 string joins columns + + + def __init__(self): + ModelDataBase.__init__(self); + + # + # Initialize with defaults. + # See the database for explanations of each of these fields. + # + self.idTestBox = None; + self.tsEffective = None; + self.tsExpire = None; + self.uidAuthor = None; + self.idGenTestBox = None; + self.ip = None; + self.uuidSystem = None; + self.sName = None; + self.idStrDescription = None; + self.fEnabled = False; + self.enmLomKind = self.ksLomKind_None; + self.ipLom = None; + self.pctScaleTimeout = 100; + self.idStrComment = None; + self.idStrOs = None; + self.idStrOsVersion = None; + self.idStrCpuVendor = None; + self.idStrCpuArch = None; + self.idStrCpuName = None; + self.lCpuRevision = None; + self.cCpus = 1; + self.fCpuHwVirt = False; + self.fCpuNestedPaging = False; + self.fCpu64BitGuest = False; + self.fChipsetIoMmu = False; + self.fRawMode = None; + self.cMbMemory = 1; + self.cMbScratch = 0; + self.idStrReport = None; + self.iTestBoxScriptRev = 0; + self.iPythonHexVersion = 0; + self.enmPendingCmd = self.ksTestBoxCmd_None; + # String table values. + self.sDescription = None; + self.sComment = None; + self.sOs = None; + self.sOsVersion = None; + self.sCpuVendor = None; + self.sCpuArch = None; + self.sCpuName = None; + self.sReport = None; + + def initFromDbRow(self, aoRow): + """ + Internal worker for initFromDbWithId and initFromDbWithGenId as well as + from TestBoxLogic. Expecting the result from a query like this: + SELECT TestBoxesWithStrings.* FROM TestBoxesWithStrings + """ + if aoRow is None: + raise TMRowNotFound('TestBox not found.'); + + self.idTestBox = aoRow[0]; + self.tsEffective = aoRow[1]; + self.tsExpire = aoRow[2]; + self.uidAuthor = aoRow[3]; + self.idGenTestBox = aoRow[4]; + self.ip = aoRow[5]; + self.uuidSystem = aoRow[6]; + self.sName = aoRow[7]; + self.idStrDescription = aoRow[8]; + self.fEnabled = aoRow[9]; + self.enmLomKind = aoRow[10]; + self.ipLom = aoRow[11]; + self.pctScaleTimeout = aoRow[12]; + self.idStrComment = aoRow[13]; + self.idStrOs = aoRow[14]; + self.idStrOsVersion = aoRow[15]; + self.idStrCpuVendor = aoRow[16]; + self.idStrCpuArch = aoRow[17]; + self.idStrCpuName = aoRow[18]; + self.lCpuRevision = aoRow[19]; + self.cCpus = aoRow[20]; + self.fCpuHwVirt = aoRow[21]; + self.fCpuNestedPaging = aoRow[22]; + self.fCpu64BitGuest = aoRow[23]; + self.fChipsetIoMmu = aoRow[24]; + self.fRawMode = aoRow[25]; + self.cMbMemory = aoRow[26]; + self.cMbScratch = aoRow[27]; + self.idStrReport = aoRow[28]; + self.iTestBoxScriptRev = aoRow[29]; + self.iPythonHexVersion = aoRow[30]; + self.enmPendingCmd = aoRow[31]; + + # String table values. + if len(aoRow) > 32: + self.sDescription = aoRow[32]; + self.sComment = aoRow[33]; + self.sOs = aoRow[34]; + self.sOsVersion = aoRow[35]; + self.sCpuVendor = aoRow[36]; + self.sCpuArch = aoRow[37]; + self.sCpuName = aoRow[38]; + self.sReport = aoRow[39]; + + return self; + + def initFromDbWithId(self, oDb, idTestBox, tsNow = None, sPeriodBack = None): + """ + Initialize the object from the database. + """ + oDb.execute(self.formatSimpleNowAndPeriodQuery(oDb, + 'SELECT TestBoxesWithStrings.*\n' + 'FROM TestBoxesWithStrings\n' + 'WHERE idTestBox = %s\n' + , ( idTestBox, ), tsNow, sPeriodBack)); + aoRow = oDb.fetchOne() + if aoRow is None: + raise TMRowNotFound('idTestBox=%s not found (tsNow=%s sPeriodBack=%s)' % (idTestBox, tsNow, sPeriodBack,)); + return self.initFromDbRow(aoRow); + + def initFromDbWithGenId(self, oDb, idGenTestBox, tsNow = None): + """ + Initialize the object from the database. + """ + _ = tsNow; # Only useful for extended data classes. + oDb.execute('SELECT TestBoxesWithStrings.*\n' + 'FROM TestBoxesWithStrings\n' + 'WHERE idGenTestBox = %s\n' + , (idGenTestBox, ) ); + return self.initFromDbRow(oDb.fetchOne()); + + def _validateAndConvertWorker(self, asAllowNullAttributes, oDb, enmValidateFor = ModelDataBase.ksValidateFor_Other): + # Override to do extra ipLom checks. + dErrors = ModelDataBase._validateAndConvertWorker(self, asAllowNullAttributes, oDb, enmValidateFor); + if self.ksParam_ipLom not in dErrors \ + and self.ksParam_enmLomKind not in dErrors \ + and self.enmLomKind != self.ksLomKind_None \ + and self.ipLom is None: + dErrors[self.ksParam_ipLom] = 'Light-out-management IP is mandatory and a LOM is selected.' + return dErrors; + + @staticmethod + def formatPythonVersionEx(iPythonHexVersion): + """ Unbuttons the version number and formats it as a version string. """ + if iPythonHexVersion is None: + return 'N/A'; + return 'v%d.%d.%d.%d' \ + % ( iPythonHexVersion >> 24, + (iPythonHexVersion >> 16) & 0xff, + (iPythonHexVersion >> 8) & 0xff, + iPythonHexVersion & 0xff); + + def formatPythonVersion(self): + """ Unbuttons the version number and formats it as a version string. """ + return self.formatPythonVersionEx(self.iPythonHexVersion); + + + @staticmethod + def getCpuFamilyEx(lCpuRevision): + """ Returns the CPU family for a x86 or amd64 testboxes.""" + if lCpuRevision is None: + return 0; + return (lCpuRevision >> 24 & 0xff); + + def getCpuFamily(self): + """ Returns the CPU family for a x86 or amd64 testboxes.""" + return self.getCpuFamilyEx(self.lCpuRevision); + + @staticmethod + def getCpuModelEx(lCpuRevision): + """ Returns the CPU model for a x86 or amd64 testboxes.""" + if lCpuRevision is None: + return 0; + return (lCpuRevision >> 8 & 0xffff); + + def getCpuModel(self): + """ Returns the CPU model for a x86 or amd64 testboxes.""" + return self.getCpuModelEx(self.lCpuRevision); + + @staticmethod + def getCpuSteppingEx(lCpuRevision): + """ Returns the CPU stepping for a x86 or amd64 testboxes.""" + if lCpuRevision is None: + return 0; + return (lCpuRevision & 0xff); + + def getCpuStepping(self): + """ Returns the CPU stepping for a x86 or amd64 testboxes.""" + return self.getCpuSteppingEx(self.lCpuRevision); + + + # The following is a translation of the g_aenmIntelFamily06 array in CPUMR3CpuId.cpp: + kdIntelFamily06 = { + 0x00: 'P6', + 0x01: 'P6', + 0x03: 'P6_II', + 0x05: 'P6_II', + 0x06: 'P6_II', + 0x07: 'P6_III', + 0x08: 'P6_III', + 0x09: 'P6_M_Banias', + 0x0a: 'P6_III', + 0x0b: 'P6_III', + 0x0d: 'P6_M_Dothan', + 0x0e: 'Core_Yonah', + 0x0f: 'Core2_Merom', + 0x15: 'P6_M_Dothan', + 0x16: 'Core2_Merom', + 0x17: 'Core2_Penryn', + 0x1a: 'Core7_Nehalem', + 0x1c: 'Atom_Bonnell', + 0x1d: 'Core2_Penryn', + 0x1e: 'Core7_Nehalem', + 0x1f: 'Core7_Nehalem', + 0x25: 'Core7_Westmere', + 0x26: 'Atom_Lincroft', + 0x27: 'Atom_Saltwell', + 0x2a: 'Core7_SandyBridge', + 0x2c: 'Core7_Westmere', + 0x2d: 'Core7_SandyBridge', + 0x2e: 'Core7_Nehalem', + 0x2f: 'Core7_Westmere', + 0x35: 'Atom_Saltwell', + 0x36: 'Atom_Saltwell', + 0x37: 'Atom_Silvermont', + 0x3a: 'Core7_IvyBridge', + 0x3c: 'Core7_Haswell', + 0x3d: 'Core7_Broadwell', + 0x3e: 'Core7_IvyBridge', + 0x3f: 'Core7_Haswell', + 0x45: 'Core7_Haswell', + 0x46: 'Core7_Haswell', + 0x47: 'Core7_Broadwell', + 0x4a: 'Atom_Silvermont', + 0x4c: 'Atom_Airmount', + 0x4d: 'Atom_Silvermont', + 0x4e: 'Core7_Skylake', + 0x4f: 'Core7_Broadwell', + 0x55: 'Core7_Skylake', + 0x56: 'Core7_Broadwell', + 0x5a: 'Atom_Silvermont', + 0x5c: 'Atom_Goldmont', + 0x5d: 'Atom_Silvermont', + 0x5e: 'Core7_Skylake', + 0x66: 'Core7_Cannonlake', + }; + # Also from CPUMR3CpuId.cpp, but the switch. + kdIntelFamily15 = { + 0x00: 'NB_Willamette', + 0x01: 'NB_Willamette', + 0x02: 'NB_Northwood', + 0x03: 'NB_Prescott', + 0x04: 'NB_Prescott2M', + 0x05: 'NB_Unknown', + 0x06: 'NB_CedarMill', + 0x07: 'NB_Gallatin', + }; + + @staticmethod + def queryCpuMicroarchEx(lCpuRevision, sCpuVendor): + """ Try guess the microarch name for the cpu. Returns None if we cannot. """ + if lCpuRevision is None or sCpuVendor is None: + return None; + uFam = TestBoxData.getCpuFamilyEx(lCpuRevision); + uMod = TestBoxData.getCpuModelEx(lCpuRevision); + if sCpuVendor == 'GenuineIntel': + if uFam == 6: + return TestBoxData.kdIntelFamily06.get(uMod, None); + if uFam == 15: + return TestBoxData.kdIntelFamily15.get(uMod, None); + elif sCpuVendor == 'AuthenticAMD': + if uFam == 0xf: + if uMod < 0x10: return 'K8_130nm'; + if 0x60 <= uMod < 0x80: return 'K8_65nm'; + if uMod >= 0x40: return 'K8_90nm_AMDV'; + if uMod in [0x21, 0x23, 0x2b, 0x37, 0x3f]: return 'K8_90nm_DualCore'; + return 'AMD_K8_90nm'; + if uFam == 0x10: return 'K10'; + if uFam == 0x11: return 'K10_Lion'; + if uFam == 0x12: return 'K10_Llano'; + if uFam == 0x14: return 'Bobcat'; + if uFam == 0x15: + if uMod <= 0x01: return 'Bulldozer'; + if uMod in [0x02, 0x10, 0x13]: return 'Piledriver'; + return None; + if uFam == 0x16: + return 'Jaguar'; + elif sCpuVendor == 'CentaurHauls': + if uFam == 0x05: + if uMod == 0x01: return 'Centaur_C6'; + if uMod == 0x04: return 'Centaur_C6'; + if uMod == 0x08: return 'Centaur_C2'; + if uMod == 0x09: return 'Centaur_C3'; + if uFam == 0x06: + if uMod == 0x05: return 'VIA_C3_M2'; + if uMod == 0x06: return 'VIA_C3_C5A'; + if uMod == 0x07: return 'VIA_C3_C5B' if TestBoxData.getCpuSteppingEx(lCpuRevision) < 8 else 'VIA_C3_C5C'; + if uMod == 0x08: return 'VIA_C3_C5N'; + if uMod == 0x09: return 'VIA_C3_C5XL' if TestBoxData.getCpuSteppingEx(lCpuRevision) < 8 else 'VIA_C3_C5P'; + if uMod == 0x0a: return 'VIA_C7_C5J'; + if uMod == 0x0f: return 'VIA_Isaiah'; + elif sCpuVendor == ' Shanghai ': + if uFam == 0x07: + if uMod == 0x0b: return 'Shanghai_KX-5000'; + return None; + + def queryCpuMicroarch(self): + """ Try guess the microarch name for the cpu. Returns None if we cannot. """ + return self.queryCpuMicroarchEx(self.lCpuRevision, self.sCpuVendor); + + @staticmethod + def getPrettyCpuVersionEx(lCpuRevision, sCpuVendor): + """ Pretty formatting of the family/model/stepping with microarch optimizations. """ + if lCpuRevision is None or sCpuVendor is None: + return u'<none>'; + sMarch = TestBoxData.queryCpuMicroarchEx(lCpuRevision, sCpuVendor); + if sMarch is not None: + return '%s %02x:%x' \ + % (sMarch, TestBoxData.getCpuModelEx(lCpuRevision), TestBoxData.getCpuSteppingEx(lCpuRevision)); + return 'fam%02X m%02X s%02X' \ + % ( TestBoxData.getCpuFamilyEx(lCpuRevision), TestBoxData.getCpuModelEx(lCpuRevision), + TestBoxData.getCpuSteppingEx(lCpuRevision)); + + def getPrettyCpuVersion(self): + """ Pretty formatting of the family/model/stepping with microarch optimizations. """ + return self.getPrettyCpuVersionEx(self.lCpuRevision, self.sCpuVendor); + + def getArchBitString(self): + """ Returns 32-bit, 64-bit, <none>, or sCpuArch. """ + if self.sCpuArch is None: + return '<none>'; + if self.sCpuArch in [ 'x86',]: + return '32-bit'; + if self.sCpuArch in [ 'amd64',]: + return '64-bit'; + return self.sCpuArch; + + def getPrettyCpuVendor(self): + """ Pretty vendor name.""" + if self.sCpuVendor is None: + return '<none>'; + if self.sCpuVendor == 'GenuineIntel': return 'Intel'; + if self.sCpuVendor == 'AuthenticAMD': return 'AMD'; + if self.sCpuVendor == 'CentaurHauls': return 'VIA'; + if self.sCpuVendor == ' Shanghai ': return 'Shanghai'; + return self.sCpuVendor; + + +class TestBoxDataEx(TestBoxData): + """ + TestBox data. + """ + + ksParam_aoInSchedGroups = 'TestBox_aoInSchedGroups'; + + # Use [] instead of None. + kasAltArrayNull = [ 'aoInSchedGroups', ]; + + ## Helper parameter containing the comma separated list with the IDs of + # potential members found in the parameters. + ksParam_aidSchedGroups = 'TestBoxDataEx_aidSchedGroups'; + + def __init__(self): + TestBoxData.__init__(self); + self.aoInSchedGroups = [] # type: list[TestBoxInSchedGroupData] + + def _initExtraMembersFromDb(self, oDb, tsNow = None, sPeriodBack = None): + """ + Worker shared by the initFromDb* methods. + Returns self. Raises exception if no row or database error. + """ + oDb.execute(self.formatSimpleNowAndPeriodQuery(oDb, + 'SELECT *\n' + 'FROM TestBoxesInSchedGroups\n' + 'WHERE idTestBox = %s\n' + , (self.idTestBox,), tsNow, sPeriodBack) + + 'ORDER BY idSchedGroup\n' ); + self.aoInSchedGroups = []; + for aoRow in oDb.fetchAll(): + self.aoInSchedGroups.append(TestBoxInSchedGroupDataEx().initFromDbRowEx(aoRow, oDb, tsNow, sPeriodBack)); + return self; + + def initFromDbRowEx(self, aoRow, oDb, tsNow = None): + """ + Reinitialize from a SELECT * FROM TestBoxesWithStrings row. Will query the + necessary additional data from oDb using tsNow. + Returns self. Raises exception if no row or database error. + """ + TestBoxData.initFromDbRow(self, aoRow); + return self._initExtraMembersFromDb(oDb, tsNow); + + def initFromDbWithId(self, oDb, idTestBox, tsNow = None, sPeriodBack = None): + """ + Initialize the object from the database. + """ + TestBoxData.initFromDbWithId(self, oDb, idTestBox, tsNow, sPeriodBack); + return self._initExtraMembersFromDb(oDb, tsNow, sPeriodBack); + + def initFromDbWithGenId(self, oDb, idGenTestBox, tsNow = None): + """ + Initialize the object from the database. + """ + TestBoxData.initFromDbWithGenId(self, oDb, idGenTestBox); + if tsNow is None and not oDb.isTsInfinity(self.tsExpire): + tsNow = self.tsEffective; + return self._initExtraMembersFromDb(oDb, tsNow); + + def getAttributeParamNullValues(self, sAttr): # Necessary? + if sAttr in ['aoInSchedGroups', ]: + return [[], '']; + return TestBoxData.getAttributeParamNullValues(self, sAttr); + + def convertParamToAttribute(self, sAttr, sParam, oValue, oDisp, fStrict): + """ + For dealing with the in-scheduling-group list. + """ + if sAttr != 'aoInSchedGroups': + return TestBoxData.convertParamToAttribute(self, sAttr, sParam, oValue, oDisp, fStrict); + + aoNewValues = []; + aidSelected = oDisp.getListOfIntParams(sParam, iMin = 1, iMax = 0x7ffffffe, aiDefaults = []); + asIds = oDisp.getStringParam(self.ksParam_aidSchedGroups, sDefault = '').split(','); + for idSchedGroup in asIds: + try: idSchedGroup = int(idSchedGroup); + except: pass; + oDispWrapper = self.DispWrapper(oDisp, '%s[%s][%%s]' % (TestBoxDataEx.ksParam_aoInSchedGroups, idSchedGroup,)) + oMember = TestBoxInSchedGroupData().initFromParams(oDispWrapper, fStrict = False); + if idSchedGroup in aidSelected: + aoNewValues.append(oMember); + return aoNewValues; + + def _validateAndConvertAttribute(self, sAttr, sParam, oValue, aoNilValues, fAllowNull, oDb): # pylint: disable=too-many-locals + """ + Validate special arrays and requirement expressions. + + Some special needs for the in-scheduling-group list. + """ + if sAttr != 'aoInSchedGroups': + return TestBoxData._validateAndConvertAttribute(self, sAttr, sParam, oValue, aoNilValues, fAllowNull, oDb); + + asErrors = []; + aoNewValues = []; + + # Note! We'll be returning an error dictionary instead of an string here. + dErrors = {}; + + # HACK ALERT! idTestBox might not have been validated and converted yet, but we need detect + # adding so we can ignore idTestBox being NIL when validating group memberships. + ## @todo make base.py pass us the ksValidateFor_Xxxx value. + fIsAdding = bool(self.idTestBox in [ None, -1, '-1', 'None', '' ]) + + for iInGrp, oInSchedGroup in enumerate(self.aoInSchedGroups): + oInSchedGroup = copy.copy(oInSchedGroup); + oInSchedGroup.idTestBox = self.idTestBox; + if fIsAdding: + dCurErrors = oInSchedGroup.validateAndConvertEx(['idTestBox',] + oInSchedGroup.kasAllowNullAttributes, + oDb, ModelDataBase.ksValidateFor_Add); + else: + dCurErrors = oInSchedGroup.validateAndConvert(oDb, ModelDataBase.ksValidateFor_Other); + if not dCurErrors: + pass; ## @todo figure out the ID? + else: + asErrors = []; + for sKey in dCurErrors: + asErrors.append('%s: %s' % (sKey[len('TestBoxInSchedGroup_'):], + dCurErrors[sKey] + ('{%s}' % self.idTestBox))) + dErrors[iInGrp] = '<br>\n'.join(asErrors) + aoNewValues.append(oInSchedGroup); + + for iInGrp, oInSchedGroup in enumerate(self.aoInSchedGroups): + for iInGrp2 in xrange(iInGrp + 1, len(self.aoInSchedGroups)): + if self.aoInSchedGroups[iInGrp2].idSchedGroup == oInSchedGroup.idSchedGroup: + sMsg = 'Duplicate scheduling group #%s".' % (oInSchedGroup.idSchedGroup,); + if iInGrp in dErrors: dErrors[iInGrp] += '<br>\n' + sMsg; + else: dErrors[iInGrp] = sMsg; + if iInGrp2 in dErrors: dErrors[iInGrp2] += '<br>\n' + sMsg; + else: dErrors[iInGrp2] = sMsg; + break; + + return (aoNewValues, dErrors if dErrors else None); + + +class TestBoxLogic(ModelLogicBase): + """ + TestBox logic. + """ + + kiSortColumn_sName = 1; + kiSortColumn_sOs = 2; + kiSortColumn_sOsVersion = 3; + kiSortColumn_sCpuVendor = 4; + kiSortColumn_sCpuArch = 5; + kiSortColumn_lCpuRevision = 6; + kiSortColumn_cCpus = 7; + kiSortColumn_cMbMemory = 8; + kiSortColumn_cMbScratch = 9; + kiSortColumn_fCpuNestedPaging = 10; + kiSortColumn_iTestBoxScriptRev = 11; + kiSortColumn_iPythonHexVersion = 12; + kiSortColumn_enmPendingCmd = 13; + kiSortColumn_fEnabled = 14; + kiSortColumn_enmState = 15; + kiSortColumn_tsUpdated = 16; + kcMaxSortColumns = 17; + kdSortColumnMap = { + 0: 'TestBoxesWithStrings.sName', + kiSortColumn_sName: "regexp_replace(TestBoxesWithStrings.sName,'[0-9]*', '', 'g'), " \ + "RIGHT(CONCAT(regexp_replace(TestBoxesWithStrings.sName,'[^0-9]*','', 'g'),'0'),8)::int", + -kiSortColumn_sName: "regexp_replace(TestBoxesWithStrings.sName,'[0-9]*', '', 'g') DESC, " \ + "RIGHT(CONCAT(regexp_replace(TestBoxesWithStrings.sName,'[^0-9]*','', 'g'),'0'),8)::int DESC", + kiSortColumn_sOs: 'TestBoxesWithStrings.sOs', + -kiSortColumn_sOs: 'TestBoxesWithStrings.sOs DESC', + kiSortColumn_sOsVersion: 'TestBoxesWithStrings.sOsVersion', + -kiSortColumn_sOsVersion: 'TestBoxesWithStrings.sOsVersion DESC', + kiSortColumn_sCpuVendor: 'TestBoxesWithStrings.sCpuVendor', + -kiSortColumn_sCpuVendor: 'TestBoxesWithStrings.sCpuVendor DESC', + kiSortColumn_sCpuArch: 'TestBoxesWithStrings.sCpuArch', + -kiSortColumn_sCpuArch: 'TestBoxesWithStrings.sCpuArch DESC', + kiSortColumn_lCpuRevision: 'TestBoxesWithStrings.lCpuRevision', + -kiSortColumn_lCpuRevision: 'TestBoxesWithStrings.lCpuRevision DESC', + kiSortColumn_cCpus: 'TestBoxesWithStrings.cCpus', + -kiSortColumn_cCpus: 'TestBoxesWithStrings.cCpus DESC', + kiSortColumn_cMbMemory: 'TestBoxesWithStrings.cMbMemory', + -kiSortColumn_cMbMemory: 'TestBoxesWithStrings.cMbMemory DESC', + kiSortColumn_cMbScratch: 'TestBoxesWithStrings.cMbScratch', + -kiSortColumn_cMbScratch: 'TestBoxesWithStrings.cMbScratch DESC', + kiSortColumn_fCpuNestedPaging: 'TestBoxesWithStrings.fCpuNestedPaging', + -kiSortColumn_fCpuNestedPaging: 'TestBoxesWithStrings.fCpuNestedPaging DESC', + kiSortColumn_iTestBoxScriptRev: 'TestBoxesWithStrings.iTestBoxScriptRev', + -kiSortColumn_iTestBoxScriptRev: 'TestBoxesWithStrings.iTestBoxScriptRev DESC', + kiSortColumn_iPythonHexVersion: 'TestBoxesWithStrings.iPythonHexVersion', + -kiSortColumn_iPythonHexVersion: 'TestBoxesWithStrings.iPythonHexVersion DESC', + kiSortColumn_enmPendingCmd: 'TestBoxesWithStrings.enmPendingCmd', + -kiSortColumn_enmPendingCmd: 'TestBoxesWithStrings.enmPendingCmd DESC', + kiSortColumn_fEnabled: 'TestBoxesWithStrings.fEnabled', + -kiSortColumn_fEnabled: 'TestBoxesWithStrings.fEnabled DESC', + kiSortColumn_enmState: 'TestBoxStatuses.enmState', + -kiSortColumn_enmState: 'TestBoxStatuses.enmState DESC', + kiSortColumn_tsUpdated: 'TestBoxStatuses.tsUpdated', + -kiSortColumn_tsUpdated: 'TestBoxStatuses.tsUpdated DESC', + }; + + def __init__(self, oDb): + ModelLogicBase.__init__(self, oDb); + self.dCache = None; + + def tryFetchTestBoxByUuid(self, sTestBoxUuid): + """ + Tries to fetch a testbox by its UUID alone. + """ + self._oDb.execute('SELECT TestBoxesWithStrings.*\n' + 'FROM TestBoxesWithStrings\n' + 'WHERE uuidSystem = %s\n' + ' AND tsExpire = \'infinity\'::timestamp\n' + 'ORDER BY tsEffective DESC\n', + (sTestBoxUuid,)); + if self._oDb.getRowCount() == 0: + return None; + if self._oDb.getRowCount() != 1: + raise TMTooManyRows('Database integrity error: %u hits' % (self._oDb.getRowCount(),)); + oData = TestBoxData(); + oData.initFromDbRow(self._oDb.fetchOne()); + return oData; + + def fetchForListing(self, iStart, cMaxRows, tsNow, aiSortColumns = None): + """ + Fetches testboxes for listing. + + Returns an array (list) of TestBoxDataForListing items, empty list if none. + The TestBoxDataForListing instances are just TestBoxData with two extra + members, an extra oStatus member that is either None or a TestBoxStatusData + instance, and a member tsCurrent holding CURRENT_TIMESTAMP. + + Raises exception on error. + """ + class TestBoxDataForListing(TestBoxDataEx): + """ We add two members for the listing. """ + def __init__(self): + TestBoxDataEx.__init__(self); + self.tsCurrent = None; # CURRENT_TIMESTAMP + self.oStatus = None # type: TestBoxStatusData + + from testmanager.core.testboxstatus import TestBoxStatusData; + + if not aiSortColumns: + aiSortColumns = [self.kiSortColumn_sName,]; + + if tsNow is None: + self._oDb.execute('SELECT TestBoxesWithStrings.*,\n' + ' TestBoxStatuses.*\n' + 'FROM TestBoxesWithStrings\n' + ' LEFT OUTER JOIN TestBoxStatuses\n' + ' ON TestBoxStatuses.idTestBox = TestBoxesWithStrings.idTestBox\n' + 'WHERE TestBoxesWithStrings.tsExpire = \'infinity\'::TIMESTAMP\n' + 'ORDER BY ' + (', '.join([self.kdSortColumnMap[i] for i in aiSortColumns])) + '\n' + 'LIMIT %s OFFSET %s\n' + , (cMaxRows, iStart,)); + else: + self._oDb.execute('SELECT TestBoxesWithStrings.*,\n' + ' TestBoxStatuses.*\n' + 'FROM TestBoxesWithStrings\n' + ' LEFT OUTER JOIN TestBoxStatuses\n' + ' ON TestBoxStatuses.idTestBox = TestBoxesWithStrings.idTestBox\n' + 'WHERE tsExpire > %s\n' + ' AND tsEffective <= %s\n' + 'ORDER BY ' + (', '.join([self.kdSortColumnMap[i] for i in aiSortColumns])) + '\n' + 'LIMIT %s OFFSET %s\n' + , ( tsNow, tsNow, cMaxRows, iStart,)); + + aoRows = []; + for aoOne in self._oDb.fetchAll(): + oTestBox = TestBoxDataForListing().initFromDbRowEx(aoOne, self._oDb, tsNow); + oTestBox.tsCurrent = self._oDb.getCurrentTimestamp(); + if aoOne[TestBoxData.kcDbColumns] is not None: + oTestBox.oStatus = TestBoxStatusData().initFromDbRow(aoOne[TestBoxData.kcDbColumns:]); + aoRows.append(oTestBox); + return aoRows; + + def fetchForSchedGroup(self, idSchedGroup, tsNow, aiSortColumns = None): + """ + Fetches testboxes for listing. + + Returns an array (list) of TestBoxDataForSchedGroup items, empty list if none. + + Raises exception on error. + """ + if not aiSortColumns: + aiSortColumns = [self.kiSortColumn_sName,]; + asSortColumns = [self.kdSortColumnMap[i] for i in aiSortColumns]; + asSortColumns.append('TestBoxesInSchedGroups.idTestBox'); + + if tsNow is None: + self._oDb.execute(''' +SELECT TestBoxesInSchedGroups.*, + TestBoxesWithStrings.* +FROM TestBoxesInSchedGroups + LEFT OUTER JOIN TestBoxesWithStrings + ON TestBoxesWithStrings.idTestBox = TestBoxesInSchedGroups.idTestBox + AND TestBoxesWithStrings.tsExpire = 'infinity'::TIMESTAMP +WHERE TestBoxesInSchedGroups.idSchedGroup = %s + AND TestBoxesInSchedGroups.tsExpire = 'infinity'::TIMESTAMP +ORDER BY ''' + ', '.join(asSortColumns), (idSchedGroup, )); + else: + self._oDb.execute(''' +SELECT TestBoxesInSchedGroups.*, + TestBoxesWithStrings.* +FROM TestBoxesInSchedGroups + LEFT OUTER JOIN TestBoxesWithStrings + ON TestBoxesWithStrings.idTestBox = TestBoxesInSchedGroups.idTestBox + AND TestBoxesWithStrings.tsExpire > %s + AND TestBoxesWithStrings.tsEffective <= %s +WHERE TestBoxesInSchedGroups.idSchedGroup = %s + AND TestBoxesInSchedGroups.tsExpire > %s + AND TestBoxesInSchedGroups.tsEffective <= %s +ORDER BY ''' + ', '.join(asSortColumns), (tsNow, tsNow, idSchedGroup, tsNow, tsNow, )); + + aoRows = []; + for aoOne in self._oDb.fetchAll(): + aoRows.append(TestBoxDataForSchedGroup().initFromDbRow(aoOne)); + return aoRows; + + def fetchForChangeLog(self, idTestBox, iStart, cMaxRows, tsNow): # pylint: disable=too-many-locals + """ + Fetches change log entries for a testbox. + + Returns an array of ChangeLogEntry instance and an indicator whether + there are more entries. + Raises exception on error. + """ + + ## @todo calc changes to scheduler group! + + if tsNow is None: + tsNow = self._oDb.getCurrentTimestamp(); + + self._oDb.execute('SELECT TestBoxesWithStrings.*\n' + 'FROM TestBoxesWithStrings\n' + 'WHERE TestBoxesWithStrings.tsEffective <= %s\n' + ' AND TestBoxesWithStrings.idTestBox = %s\n' + 'ORDER BY TestBoxesWithStrings.tsExpire DESC\n' + 'LIMIT %s OFFSET %s\n' + , (tsNow, idTestBox, cMaxRows + 1, iStart,)); + + aoRows = []; + for aoDbRow in self._oDb.fetchAll(): + aoRows.append(TestBoxData().initFromDbRow(aoDbRow)); + + # Calculate the changes. + aoEntries = []; + for i in xrange(0, len(aoRows) - 1): + oNew = aoRows[i]; + oOld = aoRows[i + 1]; + aoChanges = []; + for sAttr in oNew.getDataAttributes(): + if sAttr not in [ 'tsEffective', 'tsExpire', 'uidAuthor', ]: + oOldAttr = getattr(oOld, sAttr); + oNewAttr = getattr(oNew, sAttr); + if oOldAttr != oNewAttr: + if sAttr == 'sReport': + aoChanges.append(AttributeChangeEntryPre(sAttr, oNewAttr, oOldAttr, str(oNewAttr), str(oOldAttr))); + else: + aoChanges.append(AttributeChangeEntry(sAttr, oNewAttr, oOldAttr, str(oNewAttr), str(oOldAttr))); + aoEntries.append(ChangeLogEntry(oNew.uidAuthor, None, oNew.tsEffective, oNew.tsExpire, oNew, oOld, aoChanges)); + + # If we're at the end of the log, add the initial entry. + if len(aoRows) <= cMaxRows and aoRows: + oNew = aoRows[-1]; + aoEntries.append(ChangeLogEntry(oNew.uidAuthor, None, oNew.tsEffective, oNew.tsExpire, oNew, None, [])); + + UserAccountLogic(self._oDb).resolveChangeLogAuthors(aoEntries); + return (aoEntries, len(aoRows) > cMaxRows); + + def _validateAndConvertData(self, oData, enmValidateFor): + # type: (TestBoxDataEx, str) -> None + """ + Helper for addEntry and editEntry that validates the scheduling group IDs in + addtion to what's covered by the default validateAndConvert of the data object. + + Raises exception on invalid input. + """ + dDataErrors = oData.validateAndConvert(self._oDb, enmValidateFor); + if dDataErrors: + raise TMInvalidData('TestBoxLogic.addEntry: %s' % (dDataErrors,)); + if isinstance(oData, TestBoxDataEx): + if oData.aoInSchedGroups: + sSchedGrps = ', '.join('(%s)' % oCur.idSchedGroup for oCur in oData.aoInSchedGroups); + self._oDb.execute('SELECT SchedGroupIDs.idSchedGroup\n' + 'FROM (VALUES ' + sSchedGrps + ' ) AS SchedGroupIDs(idSchedGroup)\n' + ' LEFT OUTER JOIN SchedGroups\n' + ' ON SchedGroupIDs.idSchedGroup = SchedGroups.idSchedGroup\n' + ' AND SchedGroups.tsExpire = \'infinity\'::TIMESTAMP\n' + 'WHERE SchedGroups.idSchedGroup IS NULL\n'); + aaoRows = self._oDb.fetchAll(); + if aaoRows: + raise TMInvalidData('TestBoxLogic.addEntry missing scheduling groups: %s' + % (', '.join(str(aoRow[0]) for aoRow in aaoRows),)); + return None; + + def addEntry(self, oData, uidAuthor, fCommit = False): + # type: (TestBoxDataEx, int, bool) -> (int, int, datetime.datetime) + """ + Creates a testbox in the database. + Returns the testbox ID, testbox generation ID and effective timestamp + of the created testbox on success. Throws error on failure. + """ + + # + # Validate. Extra work because of missing foreign key (due to history). + # + self._validateAndConvertData(oData, oData.ksValidateFor_Add); + + # + # Do it. + # + self._oDb.callProc('TestBoxLogic_addEntry' + , ( uidAuthor, + oData.ip, # Should we allow setting the IP? + oData.uuidSystem, + oData.sName, + oData.sDescription, + oData.fEnabled, + oData.enmLomKind, + oData.ipLom, + oData.pctScaleTimeout, + oData.sComment, + oData.enmPendingCmd, ) ); + (idTestBox, idGenTestBox, tsEffective) = self._oDb.fetchOne(); + + for oInSchedGrp in oData.aoInSchedGroups: + self._oDb.callProc('TestBoxLogic_addGroupEntry', + ( uidAuthor, idTestBox, oInSchedGrp.idSchedGroup, oInSchedGrp.iSchedPriority,) ); + + self._oDb.maybeCommit(fCommit); + return (idTestBox, idGenTestBox, tsEffective); + + + def editEntry(self, oData, uidAuthor, fCommit = False): + """ + Data edit update, web UI is the primary user. + + oData is either TestBoxDataEx or TestBoxData. The latter is for enabling + Returns the new generation ID and effective date. + """ + + # + # Validate. + # + self._validateAndConvertData(oData, oData.ksValidateFor_Edit); + + # + # Get current data. + # + oOldData = TestBoxDataEx().initFromDbWithId(self._oDb, oData.idTestBox); + + # + # Do it. + # + if not oData.isEqualEx(oOldData, [ 'tsEffective', 'tsExpire', 'uidAuthor', 'aoInSchedGroups', ] + + TestBoxData.kasMachineSettableOnly ): + self._oDb.callProc('TestBoxLogic_editEntry' + , ( uidAuthor, + oData.idTestBox, + oData.ip, # Should we allow setting the IP? + oData.uuidSystem, + oData.sName, + oData.sDescription, + oData.fEnabled, + oData.enmLomKind, + oData.ipLom, + oData.pctScaleTimeout, + oData.sComment, + oData.enmPendingCmd, )); + (idGenTestBox, tsEffective) = self._oDb.fetchOne(); + else: + idGenTestBox = oOldData.idGenTestBox; + tsEffective = oOldData.tsEffective; + + if isinstance(oData, TestBoxDataEx): + # Calc in-group changes. + aoRemoved = list(oOldData.aoInSchedGroups); + aoNew = []; + aoUpdated = []; + for oNewInGroup in oData.aoInSchedGroups: + oOldInGroup = None; + for iCur, oCur in enumerate(aoRemoved): + if oCur.idSchedGroup == oNewInGroup.idSchedGroup: + oOldInGroup = aoRemoved.pop(iCur); + break; + if oOldInGroup is None: + aoNew.append(oNewInGroup); + elif oNewInGroup.iSchedPriority != oOldInGroup.iSchedPriority: + aoUpdated.append(oNewInGroup); + + # Remove in-groups. + for oInGroup in aoRemoved: + self._oDb.callProc('TestBoxLogic_removeGroupEntry', (uidAuthor, oData.idTestBox, oInGroup.idSchedGroup, )); + + # Add new ones. + for oInGroup in aoNew: + self._oDb.callProc('TestBoxLogic_addGroupEntry', + ( uidAuthor, oData.idTestBox, oInGroup.idSchedGroup, oInGroup.iSchedPriority, ) ); + + # Edit existing ones. + for oInGroup in aoUpdated: + self._oDb.callProc('TestBoxLogic_editGroupEntry', + ( uidAuthor, oData.idTestBox, oInGroup.idSchedGroup, oInGroup.iSchedPriority, ) ); + else: + assert isinstance(oData, TestBoxData); + + self._oDb.maybeCommit(fCommit); + return (idGenTestBox, tsEffective); + + + def removeEntry(self, uidAuthor, idTestBox, fCascade = False, fCommit = False): + """ + Delete test box and scheduling group associations. + """ + self._oDb.callProc('TestBoxLogic_removeEntry' + , ( uidAuthor, idTestBox, fCascade,)); + self._oDb.maybeCommit(fCommit); + return True; + + + def updateOnSignOn(self, idTestBox, idGenTestBox, sTestBoxAddr, sOs, sOsVersion, # pylint: disable=too-many-arguments,too-many-locals + sCpuVendor, sCpuArch, sCpuName, lCpuRevision, cCpus, fCpuHwVirt, fCpuNestedPaging, fCpu64BitGuest, + fChipsetIoMmu, fRawMode, cMbMemory, cMbScratch, sReport, iTestBoxScriptRev, iPythonHexVersion): + """ + Update the testbox attributes automatically on behalf of the testbox script. + Returns the new generation id on success, raises an exception on failure. + """ + _ = idGenTestBox; + self._oDb.callProc('TestBoxLogic_updateOnSignOn' + , ( idTestBox, + sTestBoxAddr, + sOs, + sOsVersion, + sCpuVendor, + sCpuArch, + sCpuName, + lCpuRevision, + cCpus, + fCpuHwVirt, + fCpuNestedPaging, + fCpu64BitGuest, + fChipsetIoMmu, + fRawMode, + cMbMemory, + cMbScratch, + sReport, + iTestBoxScriptRev, + iPythonHexVersion,)); + return self._oDb.fetchOne()[0]; + + + def setCommand(self, idTestBox, sOldCommand, sNewCommand, uidAuthor = None, fCommit = False, sComment = None): + """ + Sets or resets the pending command on a testbox. + Returns (idGenTestBox, tsEffective) of the new row. + """ + ## @todo throw TMInFligthCollision again... + self._oDb.callProc('TestBoxLogic_setCommand' + , ( uidAuthor, idTestBox, sOldCommand, sNewCommand, sComment,)); + aoRow = self._oDb.fetchOne(); + self._oDb.maybeCommit(fCommit); + return (aoRow[0], aoRow[1]); + + + def getAll(self): + """ + Retrieve list of all registered Test Box records from DB. + """ + self._oDb.execute('SELECT *\n' + 'FROM TestBoxesWithStrings\n' + 'WHERE tsExpire=\'infinity\'::timestamp\n' + 'ORDER BY sName') + + aaoRows = self._oDb.fetchAll() + aoRet = [] + for aoRow in aaoRows: + aoRet.append(TestBoxData().initFromDbRow(aoRow)) + return aoRet + + + def cachedLookup(self, idTestBox): + # type: (int) -> TestBoxDataEx + """ + Looks up the most recent TestBoxData object for idTestBox via + an object cache. + + Returns a shared TestBoxDataEx object. None if not found. + Raises exception on DB error. + """ + if self.dCache is None: + self.dCache = self._oDb.getCache('TestBoxData'); + oEntry = self.dCache.get(idTestBox, None); + if oEntry is None: + fNeedNow = False; + self._oDb.execute('SELECT TestBoxesWithStrings.*\n' + 'FROM TestBoxesWithStrings\n' + 'WHERE idTestBox = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , (idTestBox, )); + if self._oDb.getRowCount() == 0: + # Maybe it was deleted, try get the last entry. + self._oDb.execute('SELECT TestBoxesWithStrings.*\n' + 'FROM TestBoxesWithStrings\n' + 'WHERE idTestBox = %s\n' + 'ORDER BY tsExpire DESC\n' + 'LIMIT 1\n' + , (idTestBox, )); + fNeedNow = True; + elif self._oDb.getRowCount() > 1: + raise self._oDb.integrityException('%s infinity rows for %s' % (self._oDb.getRowCount(), idTestBox)); + + if self._oDb.getRowCount() == 1: + aaoRow = self._oDb.fetchOne(); + if not fNeedNow: + oEntry = TestBoxDataEx().initFromDbRowEx(aaoRow, self._oDb); + else: + oEntry = TestBoxDataEx().initFromDbRow(aaoRow); + oEntry.initFromDbRowEx(aaoRow, self._oDb, tsNow = db.dbTimestampMinusOneTick(oEntry.tsExpire)); + self.dCache[idTestBox] = oEntry; + return oEntry; + + + + # + # The virtual test sheriff interface. + # + + def hasTestBoxRecentlyBeenRebooted(self, idTestBox, cHoursBack = 2, tsNow = None): + """ + Checks if the testbox has been rebooted in the specified time period. + + This does not include already pending reboots, though under some + circumstances it may. These being the test box entry being edited for + other reasons. + + Returns True / False. + """ + if tsNow is None: + tsNow = self._oDb.getCurrentTimestamp(); + self._oDb.execute('SELECT COUNT(idTestBox)\n' + 'FROM TestBoxes\n' + 'WHERE idTestBox = %s\n' + ' AND tsExpire < %s\n' + ' AND tsExpire >= %s - interval \'%s hours\'\n' + ' AND enmPendingCmd IN (%s, %s)\n' + , ( idTestBox, tsNow, tsNow, cHoursBack, + TestBoxData.ksTestBoxCmd_Reboot, TestBoxData.ksTestBoxCmd_UpgradeAndReboot, )); + return self._oDb.fetchOne()[0] > 0; + + + def rebootTestBox(self, idTestBox, uidAuthor, sComment, sOldCommand = TestBoxData.ksTestBoxCmd_None, fCommit = False): + """ + Issues a reboot command for the given test box. + Return True on succes, False on in-flight collision. + May raise DB exception on other trouble. + """ + try: + self.setCommand(idTestBox, sOldCommand, TestBoxData.ksTestBoxCmd_Reboot, + uidAuthor = uidAuthor, fCommit = fCommit, sComment = sComment); + except TMInFligthCollision: + return False; + return True; + + + def disableTestBox(self, idTestBox, uidAuthor, sComment, fCommit = False): + """ + Disables the given test box. + + Raises exception on trouble, without rollback. + """ + oTestBox = TestBoxData().initFromDbWithId(self._oDb, idTestBox); + if oTestBox.fEnabled: + oTestBox.fEnabled = False; + if sComment is not None: + oTestBox.sComment = sComment; + self.editEntry(oTestBox, uidAuthor = uidAuthor, fCommit = fCommit); + return None; + + +# +# Unit testing. +# + +# pylint: disable=missing-docstring +class TestBoxDataTestCase(ModelDataBaseTestCase): + def setUp(self): + self.aoSamples = [TestBoxData(),]; + +if __name__ == '__main__': + unittest.main(); + # not reached. + diff --git a/src/VBox/ValidationKit/testmanager/core/testboxcontroller.py b/src/VBox/ValidationKit/testmanager/core/testboxcontroller.py new file mode 100755 index 00000000..b131aa88 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/testboxcontroller.py @@ -0,0 +1,954 @@ +# -*- coding: utf-8 -*- +# $Id: testboxcontroller.py $ + +""" +Test Manager Core - Web Server Abstraction Base Class. +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Standard python imports. +import re; +import os; +import string; # pylint: disable=deprecated-module +import sys; +import uuid; + +# Validation Kit imports. +from common import constants; +from testmanager import config; +from testmanager.core import coreconsts; +from testmanager.core.db import TMDatabaseConnection; +from testmanager.core.base import TMExceptionBase; +from testmanager.core.globalresource import GlobalResourceLogic; +from testmanager.core.testboxstatus import TestBoxStatusData, TestBoxStatusLogic; +from testmanager.core.testbox import TestBoxData, TestBoxLogic; +from testmanager.core.testresults import TestResultLogic, TestResultFileData; +from testmanager.core.testset import TestSetData, TestSetLogic; +from testmanager.core.systemlog import SystemLogData, SystemLogLogic; +from testmanager.core.schedulerbase import SchedulerBase; + +# Python 3 hacks: +if sys.version_info[0] >= 3: + long = int; # pylint: disable=redefined-builtin,invalid-name + + +class TestBoxControllerException(TMExceptionBase): + """ + Exception class for TestBoxController. + """ + pass; # pylint: disable=unnecessary-pass + + +class TestBoxController(object): # pylint: disable=too-few-public-methods + """ + TestBox Controller class. + """ + + ## Applicable testbox commands to an idle TestBox. + kasIdleCmds = [TestBoxData.ksTestBoxCmd_Reboot, + TestBoxData.ksTestBoxCmd_Upgrade, + TestBoxData.ksTestBoxCmd_UpgradeAndReboot, + TestBoxData.ksTestBoxCmd_Special]; + ## Applicable testbox commands to a busy TestBox. + kasBusyCmds = [TestBoxData.ksTestBoxCmd_Abort, TestBoxData.ksTestBoxCmd_Reboot]; + ## Commands that can be ACK'ed. + kasAckableCmds = [constants.tbresp.CMD_EXEC, constants.tbresp.CMD_ABORT, constants.tbresp.CMD_REBOOT, + constants.tbresp.CMD_UPGRADE, constants.tbresp.CMD_UPGRADE_AND_REBOOT, constants.tbresp.CMD_SPECIAL]; + ## Commands that can be NACK'ed or NOTSUP'ed. + kasNackableCmds = kasAckableCmds + [kasAckableCmds, constants.tbresp.CMD_IDLE, constants.tbresp.CMD_WAIT]; + + ## Mapping from TestBoxCmd_T to TestBoxState_T + kdCmdToState = \ + { \ + TestBoxData.ksTestBoxCmd_Abort: None, + TestBoxData.ksTestBoxCmd_Reboot: TestBoxStatusData.ksTestBoxState_Rebooting, + TestBoxData.ksTestBoxCmd_Upgrade: TestBoxStatusData.ksTestBoxState_Upgrading, + TestBoxData.ksTestBoxCmd_UpgradeAndReboot: TestBoxStatusData.ksTestBoxState_UpgradingAndRebooting, + TestBoxData.ksTestBoxCmd_Special: TestBoxStatusData.ksTestBoxState_DoingSpecialCmd, + }; + + ## Mapping from TestBoxCmd_T to TestBox responses commands. + kdCmdToTbRespCmd = \ + { + TestBoxData.ksTestBoxCmd_Abort: constants.tbresp.CMD_ABORT, + TestBoxData.ksTestBoxCmd_Reboot: constants.tbresp.CMD_REBOOT, + TestBoxData.ksTestBoxCmd_Upgrade: constants.tbresp.CMD_UPGRADE, + TestBoxData.ksTestBoxCmd_UpgradeAndReboot: constants.tbresp.CMD_UPGRADE_AND_REBOOT, + TestBoxData.ksTestBoxCmd_Special: constants.tbresp.CMD_SPECIAL, + }; + + ## Mapping from TestBox responses to TestBoxCmd_T commands. + kdTbRespCmdToCmd = \ + { + constants.tbresp.CMD_IDLE: None, + constants.tbresp.CMD_WAIT: None, + constants.tbresp.CMD_EXEC: None, + constants.tbresp.CMD_ABORT: TestBoxData.ksTestBoxCmd_Abort, + constants.tbresp.CMD_REBOOT: TestBoxData.ksTestBoxCmd_Reboot, + constants.tbresp.CMD_UPGRADE: TestBoxData.ksTestBoxCmd_Upgrade, + constants.tbresp.CMD_UPGRADE_AND_REBOOT: TestBoxData.ksTestBoxCmd_UpgradeAndReboot, + constants.tbresp.CMD_SPECIAL: TestBoxData.ksTestBoxCmd_Special, + }; + + + ## The path to the upgrade zip, relative WebServerGlueBase.getBaseUrl(). + ksUpgradeZip = 'htdocs/upgrade/VBoxTestBoxScript.zip'; + + ## Valid TestBox result values. + kasValidResults = list(constants.result.g_kasValidResults); + ## Mapping TestBox result values to TestStatus_T values. + kadTbResultToStatus = \ + { + constants.result.PASSED: TestSetData.ksTestStatus_Success, + constants.result.SKIPPED: TestSetData.ksTestStatus_Skipped, + constants.result.ABORTED: TestSetData.ksTestStatus_Aborted, + constants.result.BAD_TESTBOX: TestSetData.ksTestStatus_BadTestBox, + constants.result.FAILED: TestSetData.ksTestStatus_Failure, + constants.result.TIMED_OUT: TestSetData.ksTestStatus_TimedOut, + constants.result.REBOOTED: TestSetData.ksTestStatus_Rebooted, + }; + + + def __init__(self, oSrvGlue): + """ + Won't raise exceptions. + """ + self._oSrvGlue = oSrvGlue; + self._sAction = None; # _getStandardParams / dispatchRequest sets this later on. + self._idTestBox = None; # _getStandardParams / dispatchRequest sets this later on. + self._sTestBoxUuid = None; # _getStandardParams / dispatchRequest sets this later on. + self._sTestBoxAddr = None; # _getStandardParams / dispatchRequest sets this later on. + self._idTestSet = None; # _getStandardParams / dispatchRequest sets this later on. + self._dParams = None; # _getStandardParams / dispatchRequest sets this later on. + self._asCheckedParams = []; + self._dActions = \ + { \ + constants.tbreq.SIGNON : self._actionSignOn, + constants.tbreq.REQUEST_COMMAND_BUSY: self._actionRequestCommandBusy, + constants.tbreq.REQUEST_COMMAND_IDLE: self._actionRequestCommandIdle, + constants.tbreq.COMMAND_ACK : self._actionCommandAck, + constants.tbreq.COMMAND_NACK : self._actionCommandNack, + constants.tbreq.COMMAND_NOTSUP : self._actionCommandNotSup, + constants.tbreq.LOG_MAIN : self._actionLogMain, + constants.tbreq.UPLOAD : self._actionUpload, + constants.tbreq.XML_RESULTS : self._actionXmlResults, + constants.tbreq.EXEC_COMPLETED : self._actionExecCompleted, + }; + + def _getStringParam(self, sName, asValidValues = None, fStrip = False, sDefValue = None): + """ + Gets a string parameter (stripped). + + Raises exception if not found and no default is provided, or if the + value isn't found in asValidValues. + """ + if sName not in self._dParams: + if sDefValue is None: + raise TestBoxControllerException('%s parameter %s is missing' % (self._sAction, sName)); + return sDefValue; + sValue = self._dParams[sName]; + if fStrip: + sValue = sValue.strip(); + + if sName not in self._asCheckedParams: + self._asCheckedParams.append(sName); + + if asValidValues is not None and sValue not in asValidValues: + raise TestBoxControllerException('%s parameter %s value "%s" not in %s ' \ + % (self._sAction, sName, sValue, asValidValues)); + return sValue; + + def _getBoolParam(self, sName, fDefValue = None): + """ + Gets a boolean parameter. + + Raises exception if not found and no default is provided, or if not a + valid boolean. + """ + sValue = self._getStringParam(sName, [ 'True', 'true', '1', 'False', 'false', '0'], sDefValue = str(fDefValue)); + return sValue in ('True', 'true', '1',); + + def _getIntParam(self, sName, iMin = None, iMax = None): + """ + Gets a string parameter. + Raises exception if not found, not a valid integer, or if the value + isn't in the range defined by iMin and iMax. + """ + sValue = self._getStringParam(sName); + try: + iValue = int(sValue, 0); + except: + raise TestBoxControllerException('%s parameter %s value "%s" cannot be convert to an integer' \ + % (self._sAction, sName, sValue)); + + if (iMin is not None and iValue < iMin) \ + or (iMax is not None and iValue > iMax): + raise TestBoxControllerException('%s parameter %s value %d is out of range [%s..%s]' \ + % (self._sAction, sName, iValue, iMin, iMax)); + return iValue; + + def _getLongParam(self, sName, lMin = None, lMax = None, lDefValue = None): + """ + Gets a string parameter. + Raises exception if not found, not a valid long integer, or if the value + isn't in the range defined by lMin and lMax. + """ + sValue = self._getStringParam(sName, sDefValue = (str(lDefValue) if lDefValue is not None else None)); + try: + lValue = long(sValue, 0); + except Exception as oXcpt: + raise TestBoxControllerException('%s parameter %s value "%s" cannot be convert to an integer (%s)' \ + % (self._sAction, sName, sValue, oXcpt)); + + if (lMin is not None and lValue < lMin) \ + or (lMax is not None and lValue > lMax): + raise TestBoxControllerException('%s parameter %s value %d is out of range [%s..%s]' \ + % (self._sAction, sName, lValue, lMin, lMax)); + return lValue; + + def _checkForUnknownParameters(self): + """ + Check if we've handled all parameters, raises exception if anything + unknown was found. + """ + + if len(self._asCheckedParams) != len(self._dParams): + sUnknownParams = ''; + for sKey in self._dParams: + if sKey not in self._asCheckedParams: + sUnknownParams += ' ' + sKey + '=' + self._dParams[sKey]; + raise TestBoxControllerException('Unknown parameters: ' + sUnknownParams); + + return True; + + def _writeResponse(self, dParams): + """ + Makes a reply to the testbox script. + Will raise exception on failure. + """ + self._oSrvGlue.writeParams(dParams); + self._oSrvGlue.flush(); + return True; + + def _resultResponse(self, sResultValue): + """ + Makes a simple reply to the testbox script. + Will raise exception on failure. + """ + return self._writeResponse({constants.tbresp.ALL_PARAM_RESULT: sResultValue}); + + + def _idleResponse(self): + """ + Makes an IDLE reply to the testbox script. + Will raise exception on failure. + """ + return self._writeResponse({ constants.tbresp.ALL_PARAM_RESULT: constants.tbresp.CMD_IDLE }); + + def _cleanupOldTest(self, oDb, oStatusData): + """ + Cleans up any old test set that may be left behind and changes the + state to 'idle'. See scenario #9: + file://../../docs/AutomaticTestingRevamp.html#cleaning-up-abandoned-testcase + + Note. oStatusData.enmState is set to idle, but tsUpdated is not changed. + """ + + # Cleanup any abandoned test. + if oStatusData.idTestSet is not None: + SystemLogLogic(oDb).addEntry(SystemLogData.ksEvent_TestSetAbandoned, + "idTestSet=%u idTestBox=%u enmState=%s %s" + % (oStatusData.idTestSet, oStatusData.idTestBox, + oStatusData.enmState, self._sAction), + fCommit = False); + TestSetLogic(oDb).completeAsAbandoned(oStatusData.idTestSet, fCommit = False); + GlobalResourceLogic(oDb).freeGlobalResourcesByTestBox(self._idTestBox, fCommit = False); + + # Change to idle status + if oStatusData.enmState != TestBoxStatusData.ksTestBoxState_Idle: + TestBoxStatusLogic(oDb).updateState(self._idTestBox, TestBoxStatusData.ksTestBoxState_Idle, fCommit = False); + oStatusData.tsUpdated = oDb.getCurrentTimestamp(); + oStatusData.enmState = TestBoxStatusData.ksTestBoxState_Idle; + + # Commit. + oDb.commit(); + + return True; + + def _connectToDbAndValidateTb(self, asValidStates = None): + """ + Connects to the database and validates the testbox. + + Returns (TMDatabaseConnection, TestBoxStatusData, TestBoxData) on success. + Returns (None, None, None) on failure after sending the box an appropriate response. + May raise exception on DB error. + """ + oDb = TMDatabaseConnection(self._oSrvGlue.dprint); + oLogic = TestBoxStatusLogic(oDb); + (oStatusData, oTestBoxData) = oLogic.tryFetchStatusAndConfig(self._idTestBox, self._sTestBoxUuid, self._sTestBoxAddr); + if oStatusData is None: + self._resultResponse(constants.tbresp.STATUS_DEAD); + elif asValidStates is not None and oStatusData.enmState not in asValidStates: + self._resultResponse(constants.tbresp.STATUS_NACK); + elif self._idTestSet is not None and self._idTestSet != oStatusData.idTestSet: + self._resultResponse(constants.tbresp.STATUS_NACK); + else: + return (oDb, oStatusData, oTestBoxData); + return (None, None, None); + + def writeToMainLog(self, oTestSet, sText, fIgnoreSizeCheck = False): + """ Writes the text to the main log file. """ + + # Calc the file name and open the file. + sFile = os.path.join(config.g_ksFileAreaRootDir, oTestSet.sBaseFilename + '-main.log'); + if not os.path.exists(os.path.dirname(sFile)): + os.makedirs(os.path.dirname(sFile), 0o755); + + with open(sFile, 'ab') as oFile: + # Check the size. + fSizeOk = True; + if not fIgnoreSizeCheck: + oStat = os.fstat(oFile.fileno()); + fSizeOk = oStat.st_size / (1024 * 1024) < config.g_kcMbMaxMainLog; + + # Write the text. + if fSizeOk: + if sys.version_info[0] >= 3: + oFile.write(bytes(sText, 'utf-8')); + else: + oFile.write(sText); + + return fSizeOk; + + def _actionSignOn(self): # pylint: disable=too-many-locals + """ Implement sign-on """ + + # + # Validate parameters (raises exception on failure). + # + sOs = self._getStringParam(constants.tbreq.SIGNON_PARAM_OS, coreconsts.g_kasOses); + sOsVersion = self._getStringParam(constants.tbreq.SIGNON_PARAM_OS_VERSION); + sCpuVendor = self._getStringParam(constants.tbreq.SIGNON_PARAM_CPU_VENDOR); + sCpuArch = self._getStringParam(constants.tbreq.SIGNON_PARAM_CPU_ARCH, coreconsts.g_kasCpuArches); + sCpuName = self._getStringParam(constants.tbreq.SIGNON_PARAM_CPU_NAME, fStrip = True, sDefValue = ''); # new + lCpuRevision = self._getLongParam( constants.tbreq.SIGNON_PARAM_CPU_REVISION, lMin = 0, lDefValue = 0); # new + cCpus = self._getIntParam( constants.tbreq.SIGNON_PARAM_CPU_COUNT, 1, 16384); + fCpuHwVirt = self._getBoolParam( constants.tbreq.SIGNON_PARAM_HAS_HW_VIRT); + fCpuNestedPaging = self._getBoolParam( constants.tbreq.SIGNON_PARAM_HAS_NESTED_PAGING); + fCpu64BitGuest = self._getBoolParam( constants.tbreq.SIGNON_PARAM_HAS_64_BIT_GUEST, fDefValue = True); + fChipsetIoMmu = self._getBoolParam( constants.tbreq.SIGNON_PARAM_HAS_IOMMU); + fRawMode = self._getBoolParam( constants.tbreq.SIGNON_PARAM_WITH_RAW_MODE, fDefValue = None); + cMbMemory = self._getLongParam( constants.tbreq.SIGNON_PARAM_MEM_SIZE, 8, 1073741823); # 8MB..1PB + cMbScratch = self._getLongParam( constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE, 0, 1073741823); # 0..1PB + sReport = self._getStringParam(constants.tbreq.SIGNON_PARAM_REPORT, fStrip = True, sDefValue = ''); # new + iTestBoxScriptRev = self._getIntParam( constants.tbreq.SIGNON_PARAM_SCRIPT_REV, 1, 100000000); + iPythonHexVersion = self._getIntParam( constants.tbreq.SIGNON_PARAM_PYTHON_VERSION, 0x020300f0, 0x030f00f0); + self._checkForUnknownParameters(); + + # Null conversions for new parameters. + if not sReport: + sReport = None; + if not sCpuName: + sCpuName = None; + if lCpuRevision <= 0: + lCpuRevision = None; + + # + # Connect to the database and validate the testbox. + # + oDb = TMDatabaseConnection(self._oSrvGlue.dprint); + oTestBoxLogic = TestBoxLogic(oDb); + oTestBox = oTestBoxLogic.tryFetchTestBoxByUuid(self._sTestBoxUuid); + if oTestBox is None: + oSystemLogLogic = SystemLogLogic(oDb); + oSystemLogLogic.addEntry(SystemLogData.ksEvent_TestBoxUnknown, + 'addr=%s uuid=%s os=%s %d cpus' \ + % (self._sTestBoxAddr, self._sTestBoxUuid, sOs, cCpus), + 24, fCommit = True); + return self._resultResponse(constants.tbresp.STATUS_NACK); + + # + # Update the row in TestBoxes if something changed. + # + if oTestBox.cMbScratch is not None and oTestBox.cMbScratch != 0: + cPctScratchDiff = (cMbScratch - oTestBox.cMbScratch) * 100 / oTestBox.cMbScratch; + else: + cPctScratchDiff = 100; + + # pylint: disable=too-many-boolean-expressions + if self._sTestBoxAddr != oTestBox.ip \ + or sOs != oTestBox.sOs \ + or sOsVersion != oTestBox.sOsVersion \ + or sCpuVendor != oTestBox.sCpuVendor \ + or sCpuArch != oTestBox.sCpuArch \ + or sCpuName != oTestBox.sCpuName \ + or lCpuRevision != oTestBox.lCpuRevision \ + or cCpus != oTestBox.cCpus \ + or fCpuHwVirt != oTestBox.fCpuHwVirt \ + or fCpuNestedPaging != oTestBox.fCpuNestedPaging \ + or fCpu64BitGuest != oTestBox.fCpu64BitGuest \ + or fChipsetIoMmu != oTestBox.fChipsetIoMmu \ + or fRawMode != oTestBox.fRawMode \ + or cMbMemory != oTestBox.cMbMemory \ + or abs(cPctScratchDiff) >= min(4 + cMbScratch / 10240, 12) \ + or sReport != oTestBox.sReport \ + or iTestBoxScriptRev != oTestBox.iTestBoxScriptRev \ + or iPythonHexVersion != oTestBox.iPythonHexVersion: + oTestBoxLogic.updateOnSignOn(oTestBox.idTestBox, + oTestBox.idGenTestBox, + sTestBoxAddr = self._sTestBoxAddr, + sOs = sOs, + sOsVersion = sOsVersion, + sCpuVendor = sCpuVendor, + sCpuArch = sCpuArch, + sCpuName = sCpuName, + lCpuRevision = lCpuRevision, + cCpus = cCpus, + fCpuHwVirt = fCpuHwVirt, + fCpuNestedPaging = fCpuNestedPaging, + fCpu64BitGuest = fCpu64BitGuest, + fChipsetIoMmu = fChipsetIoMmu, + fRawMode = fRawMode, + cMbMemory = cMbMemory, + cMbScratch = cMbScratch, + sReport = sReport, + iTestBoxScriptRev = iTestBoxScriptRev, + iPythonHexVersion = iPythonHexVersion); + + # + # Update the testbox status, making sure there is a status. + # + oStatusLogic = TestBoxStatusLogic(oDb); + oStatusData = oStatusLogic.tryFetchStatus(oTestBox.idTestBox); + if oStatusData is not None: + self._cleanupOldTest(oDb, oStatusData); + else: + oStatusLogic.insertIdleStatus(oTestBox.idTestBox, oTestBox.idGenTestBox, fCommit = True); + + # + # ACK the request. + # + dResponse = \ + { + constants.tbresp.ALL_PARAM_RESULT: constants.tbresp.STATUS_ACK, + constants.tbresp.SIGNON_PARAM_ID: oTestBox.idTestBox, + constants.tbresp.SIGNON_PARAM_NAME: oTestBox.sName, + } + return self._writeResponse(dResponse); + + def _doGangCleanup(self, oDb, oStatusData): + """ + _doRequestCommand worker for handling a box in gang-cleanup. + This will check if all testboxes has completed their run, pretending to + be busy until that happens. Once all are completed, resources will be + freed and the testbox returns to idle state (we update oStatusData). + """ + oStatusLogic = TestBoxStatusLogic(oDb) + oTestSet = TestSetData().initFromDbWithId(oDb, oStatusData.idTestSet); + if oStatusLogic.isWholeGangDoneTesting(oTestSet.idTestSetGangLeader): + oDb.begin(); + + GlobalResourceLogic(oDb).freeGlobalResourcesByTestBox(self._idTestBox, fCommit = False); + TestBoxStatusLogic(oDb).updateState(self._idTestBox, TestBoxStatusData.ksTestBoxState_Idle, fCommit = False); + + oStatusData.tsUpdated = oDb.getCurrentTimestamp(); + oStatusData.enmState = TestBoxStatusData.ksTestBoxState_Idle; + + oDb.commit(); + return None; + + def _doGangGatheringTimedOut(self, oDb, oStatusData): + """ + _doRequestCommand worker for handling a box in gang-gathering-timed-out state. + This will do clean-ups similar to _cleanupOldTest and update the state likewise. + """ + oDb.begin(); + + TestSetLogic(oDb).completeAsGangGatheringTimeout(oStatusData.idTestSet, fCommit = False); + GlobalResourceLogic(oDb).freeGlobalResourcesByTestBox(self._idTestBox, fCommit = False); + TestBoxStatusLogic(oDb).updateState(self._idTestBox, TestBoxStatusData.ksTestBoxState_Idle, fCommit = False); + + oStatusData.tsUpdated = oDb.getCurrentTimestamp(); + oStatusData.enmState = TestBoxStatusData.ksTestBoxState_Idle; + + oDb.commit(); + return None; + + def _doGangGathering(self, oDb, oStatusData): + """ + _doRequestCommand worker for handling a box in gang-gathering state. + This only checks for timeout. It will update the oStatusData if a + timeout is detected, so that the box will be idle upon return. + """ + oStatusLogic = TestBoxStatusLogic(oDb); + if oStatusLogic.timeSinceLastChangeInSecs(oStatusData) > config.g_kcSecGangGathering \ + and SchedulerBase.tryCancelGangGathering(oDb, oStatusData): # <-- Updates oStatusData. + self._doGangGatheringTimedOut(oDb, oStatusData); + return None; + + def _doRequestCommand(self, fIdle): + """ + Common code for handling command request. + """ + + (oDb, oStatusData, oTestBoxData) = self._connectToDbAndValidateTb(); + if oDb is None: + return False; + + # + # Status clean up. + # + # Only when BUSY will the TestBox Script request and execute commands + # concurrently. So, it must be idle when sending REQUEST_COMMAND_IDLE. + # + if fIdle: + if oStatusData.enmState == TestBoxStatusData.ksTestBoxState_GangGathering: + self._doGangGathering(oDb, oStatusData); + elif oStatusData.enmState == TestBoxStatusData.ksTestBoxState_GangGatheringTimedOut: + self._doGangGatheringTimedOut(oDb, oStatusData); + elif oStatusData.enmState == TestBoxStatusData.ksTestBoxState_GangTesting: + dResponse = SchedulerBase.composeExecResponse(oDb, oTestBoxData.idTestBox, self._oSrvGlue.getBaseUrl()); + if dResponse is not None: + return dResponse; + elif oStatusData.enmState == TestBoxStatusData.ksTestBoxState_GangCleanup: + self._doGangCleanup(oDb, oStatusData); + elif oStatusData.enmState != TestBoxStatusData.ksTestBoxState_Idle: # (includes ksTestBoxState_GangGatheringTimedOut) + self._cleanupOldTest(oDb, oStatusData); + + # + # Check for pending command. + # + if oTestBoxData.enmPendingCmd != TestBoxData.ksTestBoxCmd_None: + asValidCmds = TestBoxController.kasIdleCmds if fIdle else TestBoxController.kasBusyCmds; + if oTestBoxData.enmPendingCmd in asValidCmds: + dResponse = { constants.tbresp.ALL_PARAM_RESULT: TestBoxController.kdCmdToTbRespCmd[oTestBoxData.enmPendingCmd] }; + if oTestBoxData.enmPendingCmd in [TestBoxData.ksTestBoxCmd_Upgrade, TestBoxData.ksTestBoxCmd_UpgradeAndReboot]: + dResponse[constants.tbresp.UPGRADE_PARAM_URL] = self._oSrvGlue.getBaseUrl() + TestBoxController.ksUpgradeZip; + return self._writeResponse(dResponse); + + if oTestBoxData.enmPendingCmd == TestBoxData.ksTestBoxCmd_Abort and fIdle: + TestBoxLogic(oDb).setCommand(self._idTestBox, sOldCommand = oTestBoxData.enmPendingCmd, + sNewCommand = TestBoxData.ksTestBoxCmd_None, fCommit = True); + + # + # If doing gang stuff, return 'CMD_WAIT'. + # + ## @todo r=bird: Why is GangTesting included here? Figure out when testing gang testing. + if oStatusData.enmState in [TestBoxStatusData.ksTestBoxState_GangGathering, + TestBoxStatusData.ksTestBoxState_GangTesting, + TestBoxStatusData.ksTestBoxState_GangCleanup]: + return self._resultResponse(constants.tbresp.CMD_WAIT); + + # + # If idling and enabled try schedule a new task. + # + if fIdle \ + and oTestBoxData.fEnabled \ + and not TestSetLogic(oDb).isTestBoxExecutingTooRapidly(oTestBoxData.idTestBox) \ + and oStatusData.enmState == TestBoxStatusData.ksTestBoxState_Idle: # (paranoia) + dResponse = SchedulerBase.scheduleNewTask(oDb, oTestBoxData, oStatusData.iWorkItem, self._oSrvGlue.getBaseUrl()); + if dResponse is not None: + return self._writeResponse(dResponse); + + # + # Touch the status row every couple of mins so we can tell that the box is alive. + # + oStatusLogic = TestBoxStatusLogic(oDb); + if oStatusData.enmState != TestBoxStatusData.ksTestBoxState_GangGathering \ + and oStatusLogic.timeSinceLastChangeInSecs(oStatusData) >= TestBoxStatusLogic.kcSecIdleTouchStatus: + oStatusLogic.touchStatus(oTestBoxData.idTestBox, fCommit = True); + + return self._idleResponse(); + + def _actionRequestCommandBusy(self): + """ Implement request for command. """ + self._checkForUnknownParameters(); + return self._doRequestCommand(False); + + def _actionRequestCommandIdle(self): + """ Implement request for command. """ + self._checkForUnknownParameters(); + return self._doRequestCommand(True); + + def _doCommandAckNck(self, sCmd): + """ Implements ACK, NACK and NACK(ENOTSUP). """ + + (oDb, _, _) = self._connectToDbAndValidateTb(); + if oDb is None: + return False; + + # + # If the command maps to a TestBoxCmd_T value, it means we have to + # check and update TestBoxes. If it's an ACK, the testbox status will + # need updating as well. + # + sPendingCmd = TestBoxController.kdTbRespCmdToCmd[sCmd]; + if sPendingCmd is not None: + oTestBoxLogic = TestBoxLogic(oDb) + oTestBoxLogic.setCommand(self._idTestBox, sOldCommand = sPendingCmd, + sNewCommand = TestBoxData.ksTestBoxCmd_None, fCommit = False); + + if self._sAction == constants.tbreq.COMMAND_ACK \ + and TestBoxController.kdCmdToState[sPendingCmd] is not None: + oStatusLogic = TestBoxStatusLogic(oDb); + oStatusLogic.updateState(self._idTestBox, TestBoxController.kdCmdToState[sPendingCmd], fCommit = False); + + # Commit the two updates. + oDb.commit(); + + # + # Log NACKs. + # + if self._sAction != constants.tbreq.COMMAND_ACK: + oSysLogLogic = SystemLogLogic(oDb); + oSysLogLogic.addEntry(SystemLogData.ksEvent_CmdNacked, + 'idTestBox=%s sCmd=%s' % (self._idTestBox, sPendingCmd), + 24, fCommit = True); + + return self._resultResponse(constants.tbresp.STATUS_ACK); + + def _actionCommandAck(self): + """ Implement command ACK'ing """ + sCmd = self._getStringParam(constants.tbreq.COMMAND_ACK_PARAM_CMD_NAME, TestBoxController.kasAckableCmds); + self._checkForUnknownParameters(); + return self._doCommandAckNck(sCmd); + + def _actionCommandNack(self): + """ Implement command NACK'ing """ + sCmd = self._getStringParam(constants.tbreq.COMMAND_ACK_PARAM_CMD_NAME, TestBoxController.kasNackableCmds); + self._checkForUnknownParameters(); + return self._doCommandAckNck(sCmd); + + def _actionCommandNotSup(self): + """ Implement command NACK(ENOTSUP)'ing """ + sCmd = self._getStringParam(constants.tbreq.COMMAND_ACK_PARAM_CMD_NAME, TestBoxController.kasNackableCmds); + self._checkForUnknownParameters(); + return self._doCommandAckNck(sCmd); + + def _actionLogMain(self): + """ Implement submitting log entries to the main log file. """ + # + # Parameter validation. + # + sBody = self._getStringParam(constants.tbreq.LOG_PARAM_BODY, fStrip = False); + if not sBody: + return self._resultResponse(constants.tbresp.STATUS_NACK); + self._checkForUnknownParameters(); + + (oDb, oStatusData, _) = self._connectToDbAndValidateTb([TestBoxStatusData.ksTestBoxState_Testing, + TestBoxStatusData.ksTestBoxState_GangTesting]); + if oStatusData is None: + return False; + + # + # Write the text to the log file. + # + oTestSet = TestSetData().initFromDbWithId(oDb, oStatusData.idTestSet); + self.writeToMainLog(oTestSet, sBody); + ## @todo Overflow is a hanging offence, need to note it and fail whatever is going on... + + # Done. + return self._resultResponse(constants.tbresp.STATUS_ACK); + + def _actionUpload(self): + """ Implement uploading of files. """ + # + # Parameter validation. + # + sName = self._getStringParam(constants.tbreq.UPLOAD_PARAM_NAME); + sMime = self._getStringParam(constants.tbreq.UPLOAD_PARAM_MIME); + sKind = self._getStringParam(constants.tbreq.UPLOAD_PARAM_KIND); + sDesc = self._getStringParam(constants.tbreq.UPLOAD_PARAM_DESC); + self._checkForUnknownParameters(); + + (oDb, oStatusData, _) = self._connectToDbAndValidateTb([TestBoxStatusData.ksTestBoxState_Testing, + TestBoxStatusData.ksTestBoxState_GangTesting]); + if oStatusData is None: + return False; + + if len(sName) > 128 or len(sName) < 3: + raise TestBoxControllerException('Invalid file name "%s"' % (sName,)); + if re.match(r'^[a-zA-Z0-9_\-(){}#@+,.=]*$', sName) is None: + raise TestBoxControllerException('Invalid file name "%s"' % (sName,)); + + if sMime not in [ 'text/plain', #'text/html', 'text/xml', + 'application/octet-stream', + 'image/png', #'image/gif', 'image/jpeg', + 'video/webm', #'video/mpeg', 'video/mpeg4-generic', + ]: + raise TestBoxControllerException('Invalid MIME type "%s"' % (sMime,)); + + if sKind not in TestResultFileData.kasKinds: + raise TestBoxControllerException('Invalid kind "%s"' % (sKind,)); + + if len(sDesc) > 256: + raise TestBoxControllerException('Invalid description "%s"' % (sDesc,)); + if not set(sDesc).issubset(set(string.printable)): + raise TestBoxControllerException('Invalid description "%s"' % (sDesc,)); + + if ('application/octet-stream', {}) != self._oSrvGlue.getContentType(): + raise TestBoxControllerException('Unexpected content type: %s; %s' % self._oSrvGlue.getContentType()); + + cbFile = self._oSrvGlue.getContentLength(); + if cbFile <= 0: + raise TestBoxControllerException('File "%s" is empty or negative in size (%s)' % (sName, cbFile)); + if (cbFile + 1048575) / 1048576 > config.g_kcMbMaxUploadSingle: + raise TestBoxControllerException('File "%s" is too big %u bytes (max %u MiB)' + % (sName, cbFile, config.g_kcMbMaxUploadSingle,)); + + # + # Write the text to the log file. + # + oTestSet = TestSetData().initFromDbWithId(oDb, oStatusData.idTestSet); + oDstFile = TestSetLogic(oDb).createFile(oTestSet, sName = sName, sMime = sMime, sKind = sKind, sDesc = sDesc, + cbFile = cbFile, fCommit = True); + + offFile = 0; + oSrcFile = self._oSrvGlue.getBodyIoStreamBinary(); + while offFile < cbFile: + cbToRead = cbFile - offFile; + if cbToRead > 256*1024: + cbToRead = 256*1024; + offFile += cbToRead; + + abBuf = oSrcFile.read(cbToRead); + oDstFile.write(abBuf); # pylint: disable=maybe-no-member + del abBuf; + + oDstFile.close(); # pylint: disable=maybe-no-member + + # Done. + return self._resultResponse(constants.tbresp.STATUS_ACK); + + def _actionXmlResults(self): + """ Implement submitting "XML" like test result stream. """ + # + # Parameter validation. + # + sXml = self._getStringParam(constants.tbreq.XML_RESULT_PARAM_BODY, fStrip = False); + self._checkForUnknownParameters(); + if not sXml: # Used for link check by vboxinstaller.py on Windows. + return self._resultResponse(constants.tbresp.STATUS_ACK); + + (oDb, oStatusData, _) = self._connectToDbAndValidateTb([TestBoxStatusData.ksTestBoxState_Testing, + TestBoxStatusData.ksTestBoxState_GangTesting]); + if oStatusData is None: + return False; + + # + # Process the XML. + # + (sError, fUnforgivable) = TestResultLogic(oDb).processXmlStream(sXml, self._idTestSet); + if sError is not None: + oTestSet = TestSetData().initFromDbWithId(oDb, oStatusData.idTestSet); + self.writeToMainLog(oTestSet, '\n!!XML error: %s\n%s\n\n' % (sError, sXml,)); + if fUnforgivable: + return self._resultResponse(constants.tbresp.STATUS_NACK); + return self._resultResponse(constants.tbresp.STATUS_ACK); + + + def _actionExecCompleted(self): + """ + Implement EXEC completion. + + Because the action is request by the worker thread of the testbox + script we cannot pass pending commands back to it like originally + planned. So, we just complete the test set and update the status. + """ + # + # Parameter validation. + # + sStatus = self._getStringParam(constants.tbreq.EXEC_COMPLETED_PARAM_RESULT, TestBoxController.kasValidResults); + self._checkForUnknownParameters(); + + (oDb, oStatusData, _) = self._connectToDbAndValidateTb([TestBoxStatusData.ksTestBoxState_Testing, + TestBoxStatusData.ksTestBoxState_GangTesting]); + if oStatusData is None: + return False; + + # + # Complete the status. + # + oDb.rollback(); + oDb.begin(); + oTestSetLogic = TestSetLogic(oDb); + idTestSetGangLeader = oTestSetLogic.complete(oStatusData.idTestSet, self.kadTbResultToStatus[sStatus], fCommit = False); + + oStatusLogic = TestBoxStatusLogic(oDb); + if oStatusData.enmState == TestBoxStatusData.ksTestBoxState_Testing: + assert idTestSetGangLeader is None; + GlobalResourceLogic(oDb).freeGlobalResourcesByTestBox(self._idTestBox); + oStatusLogic.updateState(self._idTestBox, TestBoxStatusData.ksTestBoxState_Idle, fCommit = False); + else: + assert idTestSetGangLeader is not None; + oStatusLogic.updateState(self._idTestBox, TestBoxStatusData.ksTestBoxState_GangCleanup, oStatusData.idTestSet, + fCommit = False); + if oStatusLogic.isWholeGangDoneTesting(idTestSetGangLeader): + GlobalResourceLogic(oDb).freeGlobalResourcesByTestBox(self._idTestBox); + oStatusLogic.updateState(self._idTestBox, TestBoxStatusData.ksTestBoxState_Idle, fCommit = False); + + oDb.commit(); + return self._resultResponse(constants.tbresp.STATUS_ACK); + + + + def _getStandardParams(self, dParams): + """ + Gets the standard parameters and validates them. + + The parameters are returned as a tuple: sAction, idTestBox, sTestBoxUuid. + Note! the sTextBoxId can be None if it's a SIGNON request. + + Raises TestBoxControllerException on invalid input. + """ + # + # Get the action parameter and validate it. + # + if constants.tbreq.ALL_PARAM_ACTION not in dParams: + raise TestBoxControllerException('No "%s" parameter in request (params: %s)' \ + % (constants.tbreq.ALL_PARAM_ACTION, dParams,)); + sAction = dParams[constants.tbreq.ALL_PARAM_ACTION]; + + if sAction not in self._dActions: + raise TestBoxControllerException('Unknown action "%s" in request (params: %s; action: %s)' \ + % (sAction, dParams, self._dActions)); + + # + # TestBox UUID. + # + if constants.tbreq.ALL_PARAM_TESTBOX_UUID not in dParams: + raise TestBoxControllerException('No "%s" parameter in request (params: %s)' \ + % (constants.tbreq.ALL_PARAM_TESTBOX_UUID, dParams,)); + sTestBoxUuid = dParams[constants.tbreq.ALL_PARAM_TESTBOX_UUID]; + try: + sTestBoxUuid = str(uuid.UUID(sTestBoxUuid)); + except Exception as oXcpt: + raise TestBoxControllerException('Invalid %s parameter value "%s": %s ' \ + % (constants.tbreq.ALL_PARAM_TESTBOX_UUID, sTestBoxUuid, oXcpt)); + if sTestBoxUuid == '00000000-0000-0000-0000-000000000000': + raise TestBoxControllerException('Invalid %s parameter value "%s": NULL UUID not allowed.' \ + % (constants.tbreq.ALL_PARAM_TESTBOX_UUID, sTestBoxUuid)); + + # + # TestBox ID. + # + if constants.tbreq.ALL_PARAM_TESTBOX_ID in dParams: + sTestBoxId = dParams[constants.tbreq.ALL_PARAM_TESTBOX_ID]; + try: + idTestBox = int(sTestBoxId); + if idTestBox <= 0 or idTestBox >= 0x7fffffff: + raise Exception; + except: + raise TestBoxControllerException('Bad value for "%s": "%s"' \ + % (constants.tbreq.ALL_PARAM_TESTBOX_ID, sTestBoxId)); + elif sAction == constants.tbreq.SIGNON: + idTestBox = None; + else: + raise TestBoxControllerException('No "%s" parameter in request (params: %s)' \ + % (constants.tbreq.ALL_PARAM_TESTBOX_ID, dParams,)); + + # + # Test Set ID. + # + if constants.tbreq.RESULT_PARAM_TEST_SET_ID in dParams: + sTestSetId = dParams[constants.tbreq.RESULT_PARAM_TEST_SET_ID]; + try: + idTestSet = int(sTestSetId); + if idTestSet <= 0 or idTestSet >= 0x7fffffff: + raise Exception; + except: + raise TestBoxControllerException('Bad value for "%s": "%s"' \ + % (constants.tbreq.RESULT_PARAM_TEST_SET_ID, sTestSetId)); + elif sAction not in [ constants.tbreq.XML_RESULTS, ]: ## More later. + idTestSet = None; + else: + raise TestBoxControllerException('No "%s" parameter in request (params: %s)' \ + % (constants.tbreq.RESULT_PARAM_TEST_SET_ID, dParams,)); + + # + # The testbox address. + # + sTestBoxAddr = self._oSrvGlue.getClientAddr(); + if sTestBoxAddr is None or sTestBoxAddr.strip() == '': + raise TestBoxControllerException('Invalid client address "%s"' % (sTestBoxAddr,)); + + # + # Update the list of checked parameters. + # + self._asCheckedParams.extend([constants.tbreq.ALL_PARAM_TESTBOX_UUID, constants.tbreq.ALL_PARAM_ACTION]); + if idTestBox is not None: + self._asCheckedParams.append(constants.tbreq.ALL_PARAM_TESTBOX_ID); + if idTestSet is not None: + self._asCheckedParams.append(constants.tbreq.RESULT_PARAM_TEST_SET_ID); + + return (sAction, idTestBox, sTestBoxUuid, sTestBoxAddr, idTestSet); + + def dispatchRequest(self): + """ + Dispatches the incoming request. + + Will raise TestBoxControllerException on failure. + """ + + # + # Must be a POST request. + # + try: + sMethod = self._oSrvGlue.getMethod(); + except Exception as oXcpt: + raise TestBoxControllerException('Error retriving request method: %s' % (oXcpt,)); + if sMethod != 'POST': + raise TestBoxControllerException('Error expected POST request not "%s"' % (sMethod,)); + + # + # Get the parameters and checks for duplicates. + # + try: + dParams = self._oSrvGlue.getParameters(); + except Exception as oXcpt: + raise TestBoxControllerException('Error retriving parameters: %s' % (oXcpt,)); + for sKey in dParams.keys(): + if len(dParams[sKey]) > 1: + raise TestBoxControllerException('Parameter "%s" is given multiple times: %s' % (sKey, dParams[sKey])); + dParams[sKey] = dParams[sKey][0]; + self._dParams = dParams; + + # + # Get+validate the standard action parameters and dispatch the request. + # + (self._sAction, self._idTestBox, self._sTestBoxUuid, self._sTestBoxAddr, self._idTestSet) = \ + self._getStandardParams(dParams); + return self._dActions[self._sAction](); diff --git a/src/VBox/ValidationKit/testmanager/core/testboxstatus.py b/src/VBox/ValidationKit/testmanager/core/testboxstatus.py new file mode 100755 index 00000000..86c6041f --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/testboxstatus.py @@ -0,0 +1,317 @@ +# -*- coding: utf-8 -*- +# $Id: testboxstatus.py $ + +""" +Test Manager - TestBoxStatus. +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Standard python imports. +import unittest; + +# Validation Kit imports. +from testmanager.core.base import ModelDataBase, ModelDataBaseTestCase, ModelLogicBase, TMTooManyRows, TMRowNotFound; +from testmanager.core.testbox import TestBoxData; + + +class TestBoxStatusData(ModelDataBase): + """ + TestBoxStatus Data. + """ + + ## @name TestBoxState_T + # @{ + ksTestBoxState_Idle = 'idle'; + ksTestBoxState_Testing = 'testing'; + ksTestBoxState_GangGathering = 'gang-gathering'; + ksTestBoxState_GangGatheringTimedOut = 'gang-gathering-timedout'; + ksTestBoxState_GangTesting = 'gang-testing'; + ksTestBoxState_GangCleanup = 'gang-cleanup'; + ksTestBoxState_Rebooting = 'rebooting'; + ksTestBoxState_Upgrading = 'upgrading'; + ksTestBoxState_UpgradingAndRebooting = 'upgrading-and-rebooting'; + ksTestBoxState_DoingSpecialCmd = 'doing-special-cmd'; + ## @} + + ksParam_idTestBox = 'TestBoxStatus_idTestBox'; + ksParam_idGenTestBox = 'TestBoxStatus_idGenTestBox' + ksParam_tsUpdated = 'TestBoxStatus_tsUpdated'; + ksParam_enmState = 'TestBoxStatus_enmState'; + ksParam_idTestSet = 'TestBoxStatus_idTestSet'; + ksParam_iWorkItem = 'TestBoxStatus_iWorkItem'; + + kasAllowNullAttributes = ['idTestSet', ]; + kasValidValues_enmState = \ + [ + ksTestBoxState_Idle, ksTestBoxState_Testing, ksTestBoxState_GangGathering, + ksTestBoxState_GangGatheringTimedOut, ksTestBoxState_GangTesting, ksTestBoxState_GangCleanup, + ksTestBoxState_Rebooting, ksTestBoxState_Upgrading, ksTestBoxState_UpgradingAndRebooting, + ksTestBoxState_DoingSpecialCmd, + ]; + + kcDbColumns = 6; + + def __init__(self): + ModelDataBase.__init__(self); + + # + # Initialize with defaults. + # See the database for explanations of each of these fields. + # + self.idTestBox = None; + self.idGenTestBox = None; + self.tsUpdated = None; + self.enmState = self.ksTestBoxState_Idle; + self.idTestSet = None; + self.iWorkItem = None; + + def initFromDbRow(self, aoRow): + """ + Internal worker for initFromDbWithId and initFromDbWithGenId as well as + TestBoxStatusLogic. + """ + + if aoRow is None: + raise TMRowNotFound('TestBoxStatus not found.'); + + self.idTestBox = aoRow[0]; + self.idGenTestBox = aoRow[1]; + self.tsUpdated = aoRow[2]; + self.enmState = aoRow[3]; + self.idTestSet = aoRow[4]; + self.iWorkItem = aoRow[5]; + return self; + + def initFromDbWithId(self, oDb, idTestBox): + """ + Initialize the object from the database. + """ + oDb.execute('SELECT *\n' + 'FROM TestBoxStatuses\n' + 'WHERE idTestBox = %s\n' + , (idTestBox, ) ); + return self.initFromDbRow(oDb.fetchOne()); + + def initFromDbWithGenId(self, oDb, idGenTestBox): + """ + Initialize the object from the database. + """ + oDb.execute('SELECT *\n' + 'FROM TestBoxStatuses\n' + 'WHERE idGenTestBox = %s\n' + , (idGenTestBox, ) ); + return self.initFromDbRow(oDb.fetchOne()); + + +class TestBoxStatusLogic(ModelLogicBase): + """ + TestBoxStatus logic. + """ + + ## The number of seconds between each time to call touchStatus() when + # returning CMD_IDLE. + kcSecIdleTouchStatus = 120; + + + def __init__(self, oDb): + ModelLogicBase.__init__(self, oDb); + + + def tryFetchStatus(self, idTestBox): + """ + Attempts to fetch the status of the given testbox. + + Returns a TestBoxStatusData object on success. + Returns None if no status was found. + Raises exception on other errors. + """ + self._oDb.execute('SELECT *\n' + 'FROM TestBoxStatuses\n' + 'WHERE idTestBox = %s\n', + (idTestBox,)); + if self._oDb.getRowCount() == 0: + return None; + oStatus = TestBoxStatusData(); + return oStatus.initFromDbRow(self._oDb.fetchOne()); + + def tryFetchStatusAndConfig(self, idTestBox, sTestBoxUuid, sTestBoxAddr): + """ + Tries to fetch the testbox status and current testbox config. + + Returns (TestBoxStatusData, TestBoxData) on success, (None, None) if + not found. May throw an exception on database error. + """ + self._oDb.execute('SELECT TestBoxStatuses.*,\n' + ' TestBoxesWithStrings.*\n' + 'FROM TestBoxStatuses,\n' + ' TestBoxesWithStrings\n' + 'WHERE TestBoxStatuses.idTestBox = %s\n' + ' AND TestBoxesWithStrings.idTestBox = %s\n' + ' AND TestBoxesWithStrings.tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND TestBoxesWithStrings.uuidSystem = %s\n' + ' AND TestBoxesWithStrings.ip = %s\n' + , ( idTestBox, + idTestBox, + sTestBoxUuid, + sTestBoxAddr,) ); + cRows = self._oDb.getRowCount(); + if cRows != 1: + if cRows != 0: + raise TMTooManyRows('tryFetchStatusForCommandReq got %s rows for idTestBox=%s' % (cRows, idTestBox)); + return (None, None); + aoRow = self._oDb.fetchOne(); + return (TestBoxStatusData().initFromDbRow(aoRow[:TestBoxStatusData.kcDbColumns]), + TestBoxData().initFromDbRow(aoRow[TestBoxStatusData.kcDbColumns:])); + + + def insertIdleStatus(self, idTestBox, idGenTestBox, fCommit = False): + """ + Inserts an idle status for the specified testbox. + """ + self._oDb.execute('INSERT INTO TestBoxStatuses (\n' + ' idTestBox,\n' + ' idGenTestBox,\n' + ' enmState,\n' + ' idTestSet,\n' + ' iWorkItem)\n' + 'VALUES ( %s,\n' + ' %s,\n' + ' \'idle\'::TestBoxState_T,\n' + ' NULL,\n' + ' 0)\n' + , (idTestBox, idGenTestBox) ); + self._oDb.maybeCommit(fCommit); + return True; + + def touchStatus(self, idTestBox, fCommit = False): + """ + Touches the testbox status row, i.e. sets tsUpdated to the current time. + """ + self._oDb.execute('UPDATE TestBoxStatuses\n' + 'SET tsUpdated = CURRENT_TIMESTAMP\n' + 'WHERE idTestBox = %s\n' + , (idTestBox,)); + self._oDb.maybeCommit(fCommit); + return True; + + def updateState(self, idTestBox, sNewState, idTestSet = None, fCommit = False): + """ + Updates the testbox state. + """ + self._oDb.execute('UPDATE TestBoxStatuses\n' + 'SET enmState = %s,\n' + ' idTestSet = %s,\n' + ' tsUpdated = CURRENT_TIMESTAMP\n' + 'WHERE idTestBox = %s\n', + (sNewState, idTestSet, idTestBox)); + self._oDb.maybeCommit(fCommit); + return True; + + def updateGangStatus(self, idTestSetGangLeader, sNewState, fCommit = False): + """ + Update the state of all members of a gang. + """ + self._oDb.execute('UPDATE TestBoxStatuses\n' + 'SET enmState = %s,\n' + ' tsUpdated = CURRENT_TIMESTAMP\n' + 'WHERE idTestBox IN (SELECT idTestBox\n' + ' FROM TestSets\n' + ' WHERE idTestSetGangLeader = %s)\n' + , (sNewState, idTestSetGangLeader,) ); + self._oDb.maybeCommit(fCommit); + return True; + + def updateWorkItem(self, idTestBox, iWorkItem, fCommit = False): + """ + Updates the testbox state. + """ + self._oDb.execute('UPDATE TestBoxStatuses\n' + 'SET iWorkItem = %s\n' + 'WHERE idTestBox = %s\n' + , ( iWorkItem, idTestBox,)); + self._oDb.maybeCommit(fCommit); + return True; + + def isWholeGangDoneTesting(self, idTestSetGangLeader): + """ + Checks if the whole gang is done testing. + """ + self._oDb.execute('SELECT COUNT(*)\n' + 'FROM TestBoxStatuses, TestSets\n' + 'WHERE TestBoxStatuses.idTestSet = TestSets.idTestSet\n' + ' AND TestSets.idTestSetGangLeader = %s\n' + ' AND TestBoxStatuses.enmState IN (%s, %s)\n' + , ( idTestSetGangLeader, + TestBoxStatusData.ksTestBoxState_GangGathering, + TestBoxStatusData.ksTestBoxState_GangTesting)); + return self._oDb.fetchOne()[0] == 0; + + def isTheWholeGangThere(self, idTestSetGangLeader): + """ + Checks if the whole gang is done testing. + """ + self._oDb.execute('SELECT COUNT(*)\n' + 'FROM TestBoxStatuses, TestSets\n' + 'WHERE TestBoxStatuses.idTestSet = TestSets.idTestSet\n' + ' AND TestSets.idTestSetGangLeader = %s\n' + ' AND TestBoxStatuses.enmState IN (%s, %s)\n' + , ( idTestSetGangLeader, + TestBoxStatusData.ksTestBoxState_GangGathering, + TestBoxStatusData.ksTestBoxState_GangTesting)); + return self._oDb.fetchOne()[0] == 0; + + def timeSinceLastChangeInSecs(self, oStatusData): + """ + Figures the time since the last status change. + """ + tsNow = self._oDb.getCurrentTimestamp(); + oDelta = tsNow - oStatusData.tsUpdated; + return oDelta.seconds + oDelta.days * 24 * 3600; + + +# +# Unit testing. +# + +# pylint: disable=missing-docstring +class TestBoxStatusDataTestCase(ModelDataBaseTestCase): + def setUp(self): + self.aoSamples = [TestBoxStatusData(),]; + +if __name__ == '__main__': + unittest.main(); + # not reached. + diff --git a/src/VBox/ValidationKit/testmanager/core/testcase.pgsql b/src/VBox/ValidationKit/testmanager/core/testcase.pgsql new file mode 100644 index 00000000..8d32d1c9 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/testcase.pgsql @@ -0,0 +1,275 @@ +-- $Id: testcase.pgsql $ +--- @file +-- VBox Test Manager Database Stored Procedures - TestCases. +-- + +-- +-- Copyright (C) 2012-2023 Oracle and/or its affiliates. +-- +-- This file is part of VirtualBox base platform packages, as +-- available from https://www.virtualbox.org. +-- +-- This program is free software; you can redistribute it and/or +-- modify it under the terms of the GNU General Public License +-- as published by the Free Software Foundation, in version 3 of the +-- License. +-- +-- This program is distributed in the hope that it will be useful, but +-- WITHOUT ANY WARRANTY; without even the implied warranty of +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +-- General Public License for more details. +-- +-- You should have received a copy of the GNU General Public License +-- along with this program; if not, see <https://www.gnu.org/licenses>. +-- +-- The contents of this file may alternatively be used under the terms +-- of the Common Development and Distribution License Version 1.0 +-- (CDDL), a copy of it is provided in the "COPYING.CDDL" file included +-- in the VirtualBox distribution, in which case the provisions of the +-- CDDL are applicable instead of those of the GPL. +-- +-- You may elect to license modified versions of this file under the +-- terms and conditions of either the GPL or the CDDL or both. +-- +-- SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +-- + +\set ON_ERROR_STOP 1 +\connect testmanager; + +DROP FUNCTION IF EXISTS add_testcase(INTEGER, TEXT, TEXT, BOOLEAN, INTEGER, TEXT, TEXT); +DROP FUNCTION IF EXISTS edit_testcase(INTEGER, INTEGER, TEXT, TEXT, BOOLEAN, INTEGER, TEXT, TEXT); +DROP FUNCTION IF EXISTS del_testcase(INTEGER); +DROP FUNCTION IF EXISTS TestCaseLogic_delEntry(INTEGER, INTEGER); +DROP FUNCTION IF EXISTS TestCaseLogic_addEntry(a_uidAuthor INTEGER, a_sName TEXT, a_sDescription TEXT, + a_fEnabled BOOL, a_cSecTimeout INTEGER, a_sTestBoxReqExpr TEXT, + a_sBuildReqExpr TEXT, a_sBaseCmd TEXT, a_sTestSuiteZips TEXT); +DROP FUNCTION IF EXISTS TestCaseLogic_editEntry(a_uidAuthor INTEGER, a_idTestCase INTEGER, a_sName TEXT, a_sDescription TEXT, + a_fEnabled BOOL, a_cSecTimeout INTEGER, a_sTestBoxReqExpr TEXT, + a_sBuildReqExpr TEXT, a_sBaseCmd TEXT, a_sTestSuiteZips TEXT); + +--- +-- Checks if the test case name is unique, ignoring a_idTestCaseIgnore. +-- Raises exception if duplicates are found. +-- +-- @internal +-- +CREATE OR REPLACE FUNCTION TestCaseLogic_checkUniqueName(a_sName TEXT, a_idTestCaseIgnore INTEGER) + RETURNS VOID AS $$ + DECLARE + v_cRows INTEGER; + BEGIN + SELECT COUNT(*) INTO v_cRows + FROM TestCases + WHERE sName = a_sName + AND tsExpire = 'infinity'::TIMESTAMP + AND idTestCase <> a_idTestCaseIgnore; + IF v_cRows <> 0 THEN + RAISE EXCEPTION 'Duplicate test case name "%" (% times)', a_sName, v_cRows; + END IF; + END; +$$ LANGUAGE plpgsql; + +--- +-- Check that the test case exists. +-- Raises exception if it doesn't. +-- +-- @internal +-- +CREATE OR REPLACE FUNCTION TestCaseLogic_checkExists(a_idTestCase INTEGER) RETURNS VOID AS $$ + BEGIN + IF NOT EXISTS( SELECT * + FROM TestCases + WHERE idTestCase = a_idTestCase + AND tsExpire = 'infinity'::TIMESTAMP ) THEN + RAISE EXCEPTION 'Test case with ID % does not currently exist', a_idTestCase; + END IF; + END; +$$ LANGUAGE plpgsql; + + +--- +-- Historize a row. +-- @internal +-- +CREATE OR REPLACE FUNCTION TestCaseLogic_historizeEntry(a_idTestCase INTEGER, a_tsExpire TIMESTAMP WITH TIME ZONE) + RETURNS VOID AS $$ + DECLARE + v_cUpdatedRows INTEGER; + BEGIN + UPDATE TestCases + SET tsExpire = a_tsExpire + WHERE idTestcase = a_idTestCase + AND tsExpire = 'infinity'::TIMESTAMP; + GET DIAGNOSTICS v_cUpdatedRows = ROW_COUNT; + IF v_cUpdatedRows <> 1 THEN + IF v_cUpdatedRows = 0 THEN + RAISE EXCEPTION 'Test case ID % does not currently exist', a_idTestCase; + END IF; + RAISE EXCEPTION 'Integrity error in TestCases: % current rows with idTestCase=%d', v_cUpdatedRows, a_idTestCase; + END IF; + END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE function TestCaseLogic_addEntry(a_uidAuthor INTEGER, a_sName TEXT, a_sDescription TEXT, + a_fEnabled BOOL, a_cSecTimeout INTEGER, a_sTestBoxReqExpr TEXT, + a_sBuildReqExpr TEXT, a_sBaseCmd TEXT, a_sTestSuiteZips TEXT, + a_sComment TEXT) + RETURNS INTEGER AS $$ + DECLARE + v_idTestCase INTEGER; + BEGIN + PERFORM TestCaseLogic_checkUniqueName(a_sName, -1); + + INSERT INTO TestCases (uidAuthor, sName, sDescription, fEnabled, cSecTimeout, + sTestBoxReqExpr, sBuildReqExpr, sBaseCmd, sTestSuiteZips, sComment) + VALUES (a_uidAuthor, a_sName, a_sDescription, a_fEnabled, a_cSecTimeout, + a_sTestBoxReqExpr, a_sBuildReqExpr, a_sBaseCmd, a_sTestSuiteZips, a_sComment) + RETURNING idTestcase INTO v_idTestCase; + RETURN v_idTestCase; + END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE function TestCaseLogic_editEntry(a_uidAuthor INTEGER, a_idTestCase INTEGER, a_sName TEXT, a_sDescription TEXT, + a_fEnabled BOOL, a_cSecTimeout INTEGER, a_sTestBoxReqExpr TEXT, + a_sBuildReqExpr TEXT, a_sBaseCmd TEXT, a_sTestSuiteZips TEXT, + a_sComment TEXT) + RETURNS INTEGER AS $$ + DECLARE + v_idGenTestCase INTEGER; + BEGIN + PERFORM TestCaseLogic_checkExists(a_idTestCase); + PERFORM TestCaseLogic_checkUniqueName(a_sName, a_idTestCase); + + PERFORM TestCaseLogic_historizeEntry(a_idTestCase, CURRENT_TIMESTAMP); + INSERT INTO TestCases (idTestCase, uidAuthor, sName, sDescription, fEnabled, cSecTimeout, + sTestBoxReqExpr, sBuildReqExpr, sBaseCmd, sTestSuiteZips, sComment) + VALUES (a_idTestCase, a_uidAuthor, a_sName, a_sDescription, a_fEnabled, a_cSecTimeout, + a_sTestBoxReqExpr, a_sBuildReqExpr, a_sBaseCmd, a_sTestSuiteZips, a_sComment) + RETURNING idGenTestCase INTO v_idGenTestCase; + RETURN v_idGenTestCase; + END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION TestCaseLogic_delEntry(a_uidAuthor INTEGER, a_idTestCase INTEGER, a_fCascade BOOLEAN) + RETURNS VOID AS $$ + DECLARE + v_Row TestCases%ROWTYPE; + v_tsEffective TIMESTAMP WITH TIME ZONE; + v_Rec RECORD; + v_sErrors TEXT; + BEGIN + -- + -- Check preconditions. + -- + IF a_fCascade <> TRUE THEN + IF EXISTS( SELECT * + FROM TestCaseDeps + WHERE idTestCasePreReq = a_idTestCase + AND tsExpire = 'infinity'::TIMESTAMP ) THEN + v_sErrors := ''; + FOR v_Rec IN + SELECT TestCases.idTestCase AS idTestCase, + TestCases.sName AS sName + FROM TestCaseDeps, TestCases + WHERE TestCaseDeps.idTestCasePreReq = a_idTestCase + AND TestCaseDeps.tsExpire = 'infinity'::TIMESTAMP + AND TestCases.idTestCase = TestCaseDeps.idTestCase + AND TestCases.tsExpire = 'infinity'::TIMESTAMP + LOOP + IF v_sErrors <> '' THEN + v_sErrors := v_sErrors || ', '; + END IF; + v_sErrors := v_sErrors || v_Rec.sName || ' (idTestCase=' || v_Rec.idTestCase || ')'; + END LOOP; + RAISE EXCEPTION 'Other test cases depends on test case with ID %: % ', a_idTestCase, v_sErrors; + END IF; + + IF EXISTS( SELECT * + FROM TestGroupMembers + WHERE idTestCase = a_idTestCase + AND tsExpire = 'infinity'::TIMESTAMP ) THEN + v_sErrors := ''; + FOR v_Rec IN + SELECT TestGroups.idTestGroup AS idTestGroup, + TestGroups.sName AS sName + FROM TestGroupMembers, TestGroups + WHERE TestGroupMembers.idTestCase = a_idTestCase + AND TestGroupMembers.tsExpire = 'infinity'::TIMESTAMP + AND TestGroupMembers.idTestGroup = TestGroups.idTestGroup + AND TestGroups.tsExpire = 'infinity'::TIMESTAMP + LOOP + IF v_sErrors <> '' THEN + v_sErrors := v_sErrors || ', '; + END IF; + v_sErrors := v_sErrors || v_Rec.sName || ' (idTestGroup=' || v_Rec.idTestGroup || ')'; + END LOOP; + RAISE EXCEPTION 'Test case with ID % is member of the following test group(s): % ', a_idTestCase, v_sErrors; + END IF; + END IF; + + -- + -- To preserve the information about who deleted the record, we try to + -- add a dummy record which expires immediately. I say try because of + -- the primary key, we must let the new record be valid for 1 us. :-( + -- + SELECT * INTO STRICT v_Row + FROM TestCases + WHERE idTestCase = a_idTestCase + AND tsExpire = 'infinity'::TIMESTAMP; + + v_tsEffective := CURRENT_TIMESTAMP - INTERVAL '1 microsecond'; + IF v_Row.tsEffective < v_tsEffective THEN + PERFORM TestCaseLogic_historizeEntry(a_idTestCase, v_tsEffective); + v_Row.tsEffective := v_tsEffective; + v_Row.tsExpire := CURRENT_TIMESTAMP; + v_Row.uidAuthor := a_uidAuthor; + SELECT NEXTVAL('TestCaseGenIdSeq') INTO v_Row.idGenTestCase; + INSERT INTO TestCases VALUES (v_Row.*); + ELSE + PERFORM TestCaseLogic_historizeEntry(a_idTestCase, CURRENT_TIMESTAMP); + END IF; + + -- + -- Delete arguments, test case dependencies and resource dependencies. + -- (We don't bother recording who deleted the records here since it's + -- a lot of work and sufficiently covered in the TestCases table.) + -- + UPDATE TestCaseArgs + SET tsExpire = CURRENT_TIMESTAMP + WHERE idTestCase = a_idTestCase + AND tsExpire = 'infinity'::TIMESTAMP; + + UPDATE TestCaseDeps + SET tsExpire = CURRENT_TIMESTAMP + WHERE idTestCase = a_idTestCase + AND tsExpire = 'infinity'::TIMESTAMP; + + UPDATE TestCaseGlobalRsrcDeps + SET tsExpire = CURRENT_TIMESTAMP + WHERE idTestCase = a_idTestCase + AND tsExpire = 'infinity'::TIMESTAMP; + + IF a_fCascade = TRUE THEN + UPDATE TestCaseDeps + SET tsExpire = CURRENT_TIMESTAMP + WHERE idTestCasePreReq = a_idTestCase + AND tsExpire = 'infinity'::TIMESTAMP; + + UPDATE TestGroupMembers + SET tsExpire = CURRENT_TIMESTAMP + WHERE idTestCase = a_idTestCase + AND tsExpire = 'infinity'::TIMESTAMP; + END IF; + + EXCEPTION + WHEN NO_DATA_FOUND THEN + RAISE EXCEPTION 'Test case with ID % does not currently exist', a_idTestCase; + WHEN TOO_MANY_ROWS THEN + RAISE EXCEPTION 'Integrity error in TestCases: Too many current rows for %', a_idTestCase; + END; +$$ LANGUAGE plpgsql; + diff --git a/src/VBox/ValidationKit/testmanager/core/testcase.py b/src/VBox/ValidationKit/testmanager/core/testcase.py new file mode 100755 index 00000000..b2820ff2 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/testcase.py @@ -0,0 +1,1467 @@ +# -*- coding: utf-8 -*- +# $Id: testcase.py $ +# pylint: disable=too-many-lines + +""" +Test Manager - Test Case. +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Standard python imports. +import copy; +import sys; +import unittest; + +# Validation Kit imports. +from common import utils; +from testmanager.core.base import ModelDataBase, ModelDataBaseTestCase, ModelLogicBase, TMExceptionBase, \ + TMInvalidData, TMRowNotFound, ChangeLogEntry, AttributeChangeEntry; +from testmanager.core.globalresource import GlobalResourceData; +from testmanager.core.useraccount import UserAccountLogic; + +# Python 3 hacks: +if sys.version_info[0] >= 3: + long = int; # pylint: disable=redefined-builtin,invalid-name + + + +class TestCaseGlobalRsrcDepData(ModelDataBase): + """ + Test case dependency on a global resource - data. + """ + + ksParam_idTestCase = 'TestCaseDependency_idTestCase'; + ksParam_idGlobalRsrc = 'TestCaseDependency_idGlobalRsrc'; + ksParam_tsEffective = 'TestCaseDependency_tsEffective'; + ksParam_tsExpire = 'TestCaseDependency_tsExpire'; + ksParam_uidAuthor = 'TestCaseDependency_uidAuthor'; + + kasAllowNullAttributes = ['idTestSet', ]; + + def __init__(self): + ModelDataBase.__init__(self); + + # + # Initialize with defaults. + # See the database for explanations of each of these fields. + # + self.idTestCase = None; + self.idGlobalRsrc = None; + self.tsEffective = None; + self.tsExpire = None; + self.uidAuthor = None; + + def initFromDbRow(self, aoRow): + """ + Reinitialize from a SELECT * FROM TestCaseDeps row. + """ + if aoRow is None: + raise TMRowNotFound('Test case not found.'); + + self.idTestCase = aoRow[0]; + self.idGlobalRsrc = aoRow[1]; + self.tsEffective = aoRow[2]; + self.tsExpire = aoRow[3]; + self.uidAuthor = aoRow[4]; + return self; + + +class TestCaseGlobalRsrcDepLogic(ModelLogicBase): + """ + Test case dependency on a global resources - logic. + """ + + def getTestCaseDeps(self, idTestCase, tsNow = None): + """ + Returns an array of (TestCaseGlobalRsrcDepData, GlobalResourceData) + with the global resources required by idTestCase. + Returns empty array if none found. Raises exception on database error. + + Note! Maybe a bit overkill... + """ + ## @todo This code isn't entirely kosher... Should use a DataEx with a oGlobalRsrc = GlobalResourceData(). + if tsNow is not None: + self._oDb.execute('SELECT *\n' + 'FROM TestCaseGlobalRsrcDeps, GlobalResources\n' + 'WHERE TestCaseGlobalRsrcDeps.idTestCase = %s\n' + ' AND TestCaseGlobalRsrcDeps.tsExpire > %s\n' + ' AND TestCaseGlobalRsrcDeps.tsEffective <= %s\n' + ' AND GlobalResources.idGlobalRsrc = TestCaseGlobalRsrcDeps.idGlobalRsrc\n' + ' AND GlobalResources.tsExpire > %s\n' + ' AND GlobalResources.tsEffective <= %s\n' + , (idTestCase, tsNow, tsNow, tsNow, tsNow) ); + else: + self._oDb.execute('SELECT *\n' + 'FROM TestCaseGlobalRsrcDeps, GlobalResources\n' + 'WHERE TestCaseGlobalRsrcDeps.idTestCase = %s\n' + ' AND GlobalResources.idGlobalRsrc = TestCaseGlobalRsrcDeps.idGlobalRsrc\n' + ' AND TestCaseGlobalRsrcDeps.tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND GlobalResources.tsExpire = \'infinity\'::TIMESTAMP\n' + , (idTestCase,)) + aaoRows = self._oDb.fetchAll(); + aoRet = [] + for aoRow in aaoRows: + oItem = [TestCaseDependencyData().initFromDbRow(aoRow), + GlobalResourceData().initFromDbRow(aoRow[5:])]; + aoRet.append(oItem); + + return aoRet + + def getTestCaseDepsIds(self, idTestCase, tsNow = None): + """ + Returns an array of global resources that idTestCase require. + Returns empty array if none found. Raises exception on database error. + """ + if tsNow is not None: + self._oDb.execute('SELECT idGlobalRsrc\n' + 'FROM TestCaseGlobalRsrcDeps\n' + 'WHERE TestCaseGlobalRsrcDeps.idTestCase = %s\n' + ' AND TestCaseGlobalRsrcDeps.tsExpire > %s\n' + ' AND TestCaseGlobalRsrcDeps.tsEffective <= %s\n' + , (idTestCase, tsNow, tsNow, ) ); + else: + self._oDb.execute('SELECT idGlobalRsrc\n' + 'FROM TestCaseGlobalRsrcDeps\n' + 'WHERE TestCaseGlobalRsrcDeps.idTestCase = %s\n' + ' AND TestCaseGlobalRsrcDeps.tsExpire = \'infinity\'::TIMESTAMP\n' + , (idTestCase,)) + aidGlobalRsrcs = [] + for aoRow in self._oDb.fetchAll(): + aidGlobalRsrcs.append(aoRow[0]); + return aidGlobalRsrcs; + + + def getDepGlobalResourceData(self, idTestCase, tsNow = None): + """ + Returns an array of objects of type GlobalResourceData on which the + specified test case depends on. + """ + if tsNow is None : + self._oDb.execute('SELECT GlobalResources.*\n' + 'FROM TestCaseGlobalRsrcDeps, GlobalResources\n' + 'WHERE TestCaseGlobalRsrcDeps.idTestCase = %s\n' + ' AND GlobalResources.idGlobalRsrc = TestCaseGlobalRsrcDeps.idGlobalRsrc\n' + ' AND TestCaseGlobalRsrcDeps.tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND GlobalResources.tsExpire = \'infinity\'::TIMESTAMP\n' + 'ORDER BY GlobalResources.idGlobalRsrc\n' + , (idTestCase,)) + else: + self._oDb.execute('SELECT GlobalResources.*\n' + 'FROM TestCaseGlobalRsrcDeps, GlobalResources\n' + 'WHERE TestCaseGlobalRsrcDeps.idTestCase = %s\n' + ' AND GlobalResources.idGlobalRsrc = TestCaseGlobalRsrcDeps.idGlobalRsrc\n' + ' AND TestCaseGlobalRsrcDeps.tsExpire > %s\n' + ' AND TestCaseGlobalRsrcDeps.tsExpire <= %s\n' + ' AND GlobalResources.tsExpire > %s\n' + ' AND GlobalResources.tsEffective <= %s\n' + 'ORDER BY GlobalResources.idGlobalRsrc\n' + , (idTestCase, tsNow, tsNow, tsNow, tsNow)); + + aaoRows = self._oDb.fetchAll() + aoRet = [] + for aoRow in aaoRows: + aoRet.append(GlobalResourceData().initFromDbRow(aoRow)); + + return aoRet + + +class TestCaseDependencyData(ModelDataBase): + """ + Test case dependency data + """ + + ksParam_idTestCase = 'TestCaseDependency_idTestCase'; + ksParam_idTestCasePreReq = 'TestCaseDependency_idTestCasePreReq'; + ksParam_tsEffective = 'TestCaseDependency_tsEffective'; + ksParam_tsExpire = 'TestCaseDependency_tsExpire'; + ksParam_uidAuthor = 'TestCaseDependency_uidAuthor'; + + + def __init__(self): + ModelDataBase.__init__(self); + + # + # Initialize with defaults. + # See the database for explanations of each of these fields. + # + self.idTestCase = None; + self.idTestCasePreReq = None; + self.tsEffective = None; + self.tsExpire = None; + self.uidAuthor = None; + + def initFromDbRow(self, aoRow): + """ + Reinitialize from a SELECT * FROM TestCaseDeps row. + """ + if aoRow is None: + raise TMRowNotFound('Test case not found.'); + + self.idTestCase = aoRow[0]; + self.idTestCasePreReq = aoRow[1]; + self.tsEffective = aoRow[2]; + self.tsExpire = aoRow[3]; + self.uidAuthor = aoRow[4]; + return self; + + def initFromParams(self, oDisp, fStrict=True): + """ + Initialize the object from parameters. + The input is not validated at all, except that all parameters must be + present when fStrict is True. + Note! Returns parameter NULL values, not database ones. + """ + + self.convertToParamNull(); + fn = oDisp.getStringParam; # Shorter... + + self.idTestCase = fn(self.ksParam_idTestCase, None, None if fStrict else self.idTestCase); + self.idTestCasePreReq = fn(self.ksParam_idTestCasePreReq, None, None if fStrict else self.idTestCasePreReq); + self.tsEffective = fn(self.ksParam_tsEffective, None, None if fStrict else self.tsEffective); + self.tsExpire = fn(self.ksParam_tsExpire, None, None if fStrict else self.tsExpire); + self.uidAuthor = fn(self.ksParam_uidAuthor, None, None if fStrict else self.uidAuthor); + + return True + + def validateAndConvert(self, oDb = None, enmValidateFor = ModelDataBase.ksValidateFor_Other): + """ + Validates the input and converts valid fields to their right type. + Returns a dictionary with per field reports, only invalid fields will + be returned, so an empty dictionary means that the data is valid. + + The dictionary keys are ksParam_*. + """ + dErrors = {} + + self.idTestCase = self._validateInt( dErrors, self.ksParam_idTestCase, self.idTestCase); + self.idTestCasePreReq = self._validateInt( dErrors, self.ksParam_idTestCasePreReq, self.idTestCasePreReq); + self.tsEffective = self._validateTs( dErrors, self.ksParam_tsEffective, self.tsEffective); + self.tsExpire = self._validateTs( dErrors, self.ksParam_tsExpire, self.tsExpire); + self.uidAuthor = self._validateInt( dErrors, self.ksParam_uidAuthor, self.uidAuthor); + + _ = oDb; + _ = enmValidateFor; + return dErrors + + def convertFromParamNull(self): + """ + Converts from parameter NULL values to database NULL values (None). + """ + if self.idTestCase in [-1, '']: self.idTestCase = None; + if self.idTestCasePreReq in [-1, '']: self.idTestCasePreReq = None; + if self.tsEffective == '': self.tsEffective = None; + if self.tsExpire == '': self.tsExpire = None; + if self.uidAuthor in [-1, '']: self.uidAuthor = None; + return True; + + def convertToParamNull(self): + """ + Converts from database NULL values (None) to special values we can + pass thru parameters list. + """ + if self.idTestCase is None: self.idTestCase = -1; + if self.idTestCasePreReq is None: self.idTestCasePreReq = -1; + if self.tsEffective is None: self.tsEffective = ''; + if self.tsExpire is None: self.tsExpire = ''; + if self.uidAuthor is None: self.uidAuthor = -1; + return True; + + def isEqual(self, oOther): + """ Compares two instances. """ + return self.idTestCase == oOther.idTestCase \ + and self.idTestCasePreReq == oOther.idTestCasePreReq \ + and self.tsEffective == oOther.tsEffective \ + and self.tsExpire == oOther.tsExpire \ + and self.uidAuthor == oOther.uidAuthor; + + def getTestCasePreReqIds(self, aTestCaseDependencyData): + """ + Get list of Test Case IDs which current + Test Case depends on + """ + if not aTestCaseDependencyData: + return [] + + aoRet = [] + for oTestCaseDependencyData in aTestCaseDependencyData: + aoRet.append(oTestCaseDependencyData.idTestCasePreReq) + + return aoRet + +class TestCaseDependencyLogic(ModelLogicBase): + """Test case dependency management logic""" + + def getTestCaseDeps(self, idTestCase, tsEffective = None): + """ + Returns an array of TestCaseDependencyData with the prerequisites of + idTestCase. + Returns empty array if none found. Raises exception on database error. + """ + if tsEffective is not None: + self._oDb.execute('SELECT *\n' + 'FROM TestCaseDeps\n' + 'WHERE idTestCase = %s\n' + ' AND tsExpire > %s\n' + ' AND tsEffective <= %s\n' + , (idTestCase, tsEffective, tsEffective, ) ); + else: + self._oDb.execute('SELECT *\n' + 'FROM TestCaseDeps\n' + 'WHERE idTestCase = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , (idTestCase, ) ); + aaoRows = self._oDb.fetchAll(); + aoRet = []; + for aoRow in aaoRows: + aoRet.append(TestCaseDependencyData().initFromDbRow(aoRow)); + + return aoRet + + def getTestCaseDepsIds(self, idTestCase, tsNow = None): + """ + Returns an array of test case IDs of the prerequisites of idTestCase. + Returns empty array if none found. Raises exception on database error. + """ + if tsNow is not None: + self._oDb.execute('SELECT idTestCase\n' + 'FROM TestCaseDeps\n' + 'WHERE idTestCase = %s\n' + ' AND tsExpire > %s\n' + ' AND tsEffective <= %s\n' + , (idTestCase, tsNow, tsNow, ) ); + else: + self._oDb.execute('SELECT idTestCase\n' + 'FROM TestCaseDeps\n' + 'WHERE idTestCase = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , (idTestCase, ) ); + aidPreReqs = []; + for aoRow in self._oDb.fetchAll(): + aidPreReqs.append(aoRow[0]); + return aidPreReqs; + + + def getDepTestCaseData(self, idTestCase, tsNow = None): + """ + Returns an array of objects of type TestCaseData2 on which + specified test case depends on + """ + if tsNow is None: + self._oDb.execute('SELECT TestCases.*\n' + 'FROM TestCases, TestCaseDeps\n' + 'WHERE TestCaseDeps.idTestCase = %s\n' + ' AND TestCaseDeps.idTestCasePreReq = TestCases.idTestCase\n' + ' AND TestCaseDeps.tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND TestCases.tsExpire = \'infinity\'::TIMESTAMP\n' + 'ORDER BY TestCases.idTestCase\n' + , (idTestCase, ) ); + else: + self._oDb.execute('SELECT TestCases.*\n' + 'FROM TestCases, TestCaseDeps\n' + 'WHERE TestCaseDeps.idTestCase = %s\n' + ' AND TestCaseDeps.idTestCasePreReq = TestCases.idTestCase\n' + ' AND TestCaseDeps.tsExpire > %s\n' + ' AND TestCaseDeps.tsEffective <= %s\n' + ' AND TestCases.tsExpire > %s\n' + ' AND TestCases.tsEffective <= %s\n' + 'ORDER BY TestCases.idTestCase\n' + , (idTestCase, tsNow, tsNow, tsNow, tsNow, ) ); + + aaoRows = self._oDb.fetchAll() + aoRet = [] + for aoRow in aaoRows: + aoRet.append(TestCaseData().initFromDbRow(aoRow)); + + return aoRet + + def getApplicableDepTestCaseData(self, idTestCase): + """ + Returns an array of objects of type TestCaseData on which + specified test case might depends on (all test + cases except the specified one and those testcases which are + depend on idTestCase) + """ + self._oDb.execute('SELECT *\n' + 'FROM TestCases\n' + 'WHERE idTestCase <> %s\n' + ' AND idTestCase NOT IN (SELECT idTestCase\n' + ' FROM TestCaseDeps\n' + ' WHERE idTestCasePreReq=%s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP)\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , (idTestCase, idTestCase) ) + + aaoRows = self._oDb.fetchAll() + aoRet = [] + for aoRow in aaoRows: + aoRet.append(TestCaseData().initFromDbRow(aoRow)); + + return aoRet + +class TestCaseData(ModelDataBase): + """ + Test case data + """ + + ksIdAttr = 'idTestCase'; + ksIdGenAttr = 'idGenTestCase'; + + ksParam_idTestCase = 'TestCase_idTestCase' + ksParam_tsEffective = 'TestCase_tsEffective' + ksParam_tsExpire = 'TestCase_tsExpire' + ksParam_uidAuthor = 'TestCase_uidAuthor' + ksParam_idGenTestCase = 'TestCase_idGenTestCase' + ksParam_sName = 'TestCase_sName' + ksParam_sDescription = 'TestCase_sDescription' + ksParam_fEnabled = 'TestCase_fEnabled' + ksParam_cSecTimeout = 'TestCase_cSecTimeout' + ksParam_sTestBoxReqExpr = 'TestCase_sTestBoxReqExpr'; + ksParam_sBuildReqExpr = 'TestCase_sBuildReqExpr'; + ksParam_sBaseCmd = 'TestCase_sBaseCmd' + ksParam_sValidationKitZips = 'TestCase_sValidationKitZips' + ksParam_sComment = 'TestCase_sComment' + + kasAllowNullAttributes = [ 'idTestCase', 'tsEffective', 'tsExpire', 'uidAuthor', 'idGenTestCase', 'sDescription', + 'sTestBoxReqExpr', 'sBuildReqExpr', 'sValidationKitZips', 'sComment' ]; + + kcDbColumns = 14; + + def __init__(self): + ModelDataBase.__init__(self); + + # + # Initialize with defaults. + # See the database for explanations of each of these fields. + # + self.idTestCase = None; + self.tsEffective = None; + self.tsExpire = None; + self.uidAuthor = None; + self.idGenTestCase = None; + self.sName = None; + self.sDescription = None; + self.fEnabled = False; + self.cSecTimeout = 10; # Init with minimum timeout value + self.sTestBoxReqExpr = None; + self.sBuildReqExpr = None; + self.sBaseCmd = None; + self.sValidationKitZips = None; + self.sComment = None; + + def initFromDbRow(self, aoRow): + """ + Reinitialize from a SELECT * FROM TestCases row. + Returns self. Raises exception if no row. + """ + if aoRow is None: + raise TMRowNotFound('Test case not found.'); + + self.idTestCase = aoRow[0]; + self.tsEffective = aoRow[1]; + self.tsExpire = aoRow[2]; + self.uidAuthor = aoRow[3]; + self.idGenTestCase = aoRow[4]; + self.sName = aoRow[5]; + self.sDescription = aoRow[6]; + self.fEnabled = aoRow[7]; + self.cSecTimeout = aoRow[8]; + self.sTestBoxReqExpr = aoRow[9]; + self.sBuildReqExpr = aoRow[10]; + self.sBaseCmd = aoRow[11]; + self.sValidationKitZips = aoRow[12]; + self.sComment = aoRow[13]; + return self; + + def initFromDbWithId(self, oDb, idTestCase, tsNow = None, sPeriodBack = None): + """ + Initialize the object from the database. + """ + oDb.execute(self.formatSimpleNowAndPeriodQuery(oDb, + 'SELECT *\n' + 'FROM TestCases\n' + 'WHERE idTestCase = %s\n' + , ( idTestCase,), tsNow, sPeriodBack)); + aoRow = oDb.fetchOne() + if aoRow is None: + raise TMRowNotFound('idTestCase=%s not found (tsNow=%s sPeriodBack=%s)' % (idTestCase, tsNow, sPeriodBack,)); + return self.initFromDbRow(aoRow); + + def initFromDbWithGenId(self, oDb, idGenTestCase, tsNow = None): + """ + Initialize the object from the database. + """ + _ = tsNow; # For relevant for the TestCaseDataEx version only. + oDb.execute('SELECT *\n' + 'FROM TestCases\n' + 'WHERE idGenTestCase = %s\n' + , (idGenTestCase, ) ); + return self.initFromDbRow(oDb.fetchOne()); + + def _validateAndConvertAttribute(self, sAttr, sParam, oValue, aoNilValues, fAllowNull, oDb): + if sAttr == 'cSecTimeout' and oValue not in aoNilValues: # Allow human readable interval formats. + return utils.parseIntervalSeconds(oValue); + + (oValue, sError) = ModelDataBase._validateAndConvertAttribute(self, sAttr, sParam, oValue, aoNilValues, fAllowNull, oDb); + if sError is None: + if sAttr == 'sTestBoxReqExpr': + sError = TestCaseData.validateTestBoxReqExpr(oValue); + elif sAttr == 'sBuildReqExpr': + sError = TestCaseData.validateBuildReqExpr(oValue); + elif sAttr == 'sBaseCmd': + _, sError = TestCaseData.validateStr(oValue, fAllowUnicodeSymbols=False); + return (oValue, sError); + + + # + # Misc. + # + + def needValidationKitBit(self): + """ + Predicate method for checking whether a validation kit build is required. + """ + return self.sValidationKitZips is None \ + or self.sValidationKitZips.find('@VALIDATIONKIT_ZIP@') >= 0; + + def matchesTestBoxProps(self, oTestBoxData): + """ + Checks if the all of the testbox related test requirements matches the + given testbox. + + Returns True or False according to the expression, None on exception or + non-boolean expression result. + """ + return TestCaseData.matchesTestBoxPropsEx(oTestBoxData, self.sTestBoxReqExpr); + + def matchesBuildProps(self, oBuildDataEx): + """ + Checks if the all of the build related test requirements matches the + given build. + + Returns True or False according to the expression, None on exception or + non-boolean expression result. + """ + return TestCaseData.matchesBuildPropsEx(oBuildDataEx, self.sBuildReqExpr); + + + # + # Expression validation code shared with TestCaseArgsDataEx. + # + @staticmethod + def _safelyEvalExpr(sExpr, dLocals, fMayRaiseXcpt = False): + """ + Safely evaluate requirment expression given a set of locals. + + Returns True or False according to the expression. If the expression + causes an exception to be raised or does not return a boolean result, + None will be returned. + """ + if sExpr is None or sExpr == '': + return True; + + dGlobals = \ + { + '__builtins__': None, + 'long': long, + 'int': int, + 'bool': bool, + 'True': True, + 'False': False, + 'len': len, + 'isinstance': isinstance, + 'type': type, + 'dict': dict, + 'dir': dir, + 'list': list, + 'versionCompare': utils.versionCompare, + }; + + try: + fRc = eval(sExpr, dGlobals, dLocals); + except: + if fMayRaiseXcpt: + raise; + return None; + + if not isinstance(fRc, bool): + if fMayRaiseXcpt: + raise Exception('not a boolean result: "%s" - %s' % (fRc, type(fRc)) ); + return None; + + return fRc; + + @staticmethod + def _safelyValidateReqExpr(sExpr, adLocals): + """ + Validates a requirement expression using the given sets of locals, + returning None on success and an error string on failure. + """ + for dLocals in adLocals: + try: + TestCaseData._safelyEvalExpr(sExpr, dLocals, True); + except Exception as oXcpt: + return str(oXcpt); + return None; + + @staticmethod + def validateTestBoxReqExpr(sExpr): + """ + Validates a testbox expression, returning None on success and an error + string on failure. + """ + adTestBoxes = \ + [ + { + 'sOs': 'win', + 'sOsVersion': '3.1', + 'sCpuVendor': 'VirtualBox', + 'sCpuArch': 'x86', + 'cCpus': 1, + 'fCpuHwVirt': False, + 'fCpuNestedPaging': False, + 'fCpu64BitGuest': False, + 'fChipsetIoMmu': False, + 'fRawMode': False, + 'cMbMemory': 985034, + 'cMbScratch': 1234089, + 'iTestBoxScriptRev': 1, + 'sName': 'emanon', + 'uuidSystem': '8FF81BE5-3901-4AB1-8A65-B48D511C0321', + }, + { + 'sOs': 'linux', + 'sOsVersion': '3.1', + 'sCpuVendor': 'VirtualBox', + 'sCpuArch': 'amd64', + 'cCpus': 8191, + 'fCpuHwVirt': True, + 'fCpuNestedPaging': True, + 'fCpu64BitGuest': True, + 'fChipsetIoMmu': True, + 'fRawMode': True, + 'cMbMemory': 9999999999, + 'cMbScratch': 9999999999999, + 'iTestBoxScriptRev': 9999999, + 'sName': 'emanon', + 'uuidSystem': '00000000-0000-0000-0000-000000000000', + }, + ]; + return TestCaseData._safelyValidateReqExpr(sExpr, adTestBoxes); + + @staticmethod + def matchesTestBoxPropsEx(oTestBoxData, sExpr): + """ Worker for TestCaseData.matchesTestBoxProps and TestCaseArgsDataEx.matchesTestBoxProps. """ + if sExpr is None: + return True; + dLocals = \ + { + 'sOs': oTestBoxData.sOs, + 'sOsVersion': oTestBoxData.sOsVersion, + 'sCpuVendor': oTestBoxData.sCpuVendor, + 'sCpuArch': oTestBoxData.sCpuArch, + 'iCpuFamily': oTestBoxData.getCpuFamily(), + 'iCpuModel': oTestBoxData.getCpuModel(), + 'cCpus': oTestBoxData.cCpus, + 'fCpuHwVirt': oTestBoxData.fCpuHwVirt, + 'fCpuNestedPaging': oTestBoxData.fCpuNestedPaging, + 'fCpu64BitGuest': oTestBoxData.fCpu64BitGuest, + 'fChipsetIoMmu': oTestBoxData.fChipsetIoMmu, + 'fRawMode': oTestBoxData.fRawMode, + 'cMbMemory': oTestBoxData.cMbMemory, + 'cMbScratch': oTestBoxData.cMbScratch, + 'iTestBoxScriptRev': oTestBoxData.iTestBoxScriptRev, + 'iPythonHexVersion': oTestBoxData.iPythonHexVersion, + 'sName': oTestBoxData.sName, + 'uuidSystem': oTestBoxData.uuidSystem, + }; + return TestCaseData._safelyEvalExpr(sExpr, dLocals); + + @staticmethod + def validateBuildReqExpr(sExpr): + """ + Validates a testbox expression, returning None on success and an error + string on failure. + """ + adBuilds = \ + [ + { + 'sProduct': 'VirtualBox', + 'sBranch': 'trunk', + 'sType': 'release', + 'asOsArches': ['win.amd64', 'win.x86'], + 'sVersion': '1.0', + 'iRevision': 1234, + 'uidAuthor': None, + 'idBuild': 953, + }, + { + 'sProduct': 'VirtualBox', + 'sBranch': 'VBox-4.1', + 'sType': 'release', + 'asOsArches': ['linux.x86',], + 'sVersion': '4.2.15', + 'iRevision': 89876, + 'uidAuthor': None, + 'idBuild': 945689, + }, + { + 'sProduct': 'VirtualBox', + 'sBranch': 'VBox-4.1', + 'sType': 'strict', + 'asOsArches': ['solaris.x86', 'solaris.amd64',], + 'sVersion': '4.3.0_RC3', + 'iRevision': 97939, + 'uidAuthor': 33, + 'idBuild': 9456893, + }, + ]; + return TestCaseData._safelyValidateReqExpr(sExpr, adBuilds); + + @staticmethod + def matchesBuildPropsEx(oBuildDataEx, sExpr): + """ + Checks if the all of the build related test requirements matches the + given build. + """ + if sExpr is None: + return True; + dLocals = \ + { + 'sProduct': oBuildDataEx.oCat.sProduct, + 'sBranch': oBuildDataEx.oCat.sBranch, + 'sType': oBuildDataEx.oCat.sType, + 'asOsArches': oBuildDataEx.oCat.asOsArches, + 'sVersion': oBuildDataEx.sVersion, + 'iRevision': oBuildDataEx.iRevision, + 'uidAuthor': oBuildDataEx.uidAuthor, + 'idBuild': oBuildDataEx.idBuild, + }; + return TestCaseData._safelyEvalExpr(sExpr, dLocals); + + + + +class TestCaseDataEx(TestCaseData): + """ + Test case data. + """ + + ksParam_aoTestCaseArgs = 'TestCase_aoTestCaseArgs'; + ksParam_aoDepTestCases = 'TestCase_aoDepTestCases'; + ksParam_aoDepGlobalResources = 'TestCase_aoDepGlobalResources'; + + # Use [] instead of None. + kasAltArrayNull = [ 'aoTestCaseArgs', 'aoDepTestCases', 'aoDepGlobalResources' ]; + + + def __init__(self): + TestCaseData.__init__(self); + + # List of objects of type TestCaseData (or TestCaseDataEx, we don't + # care) on which current Test Case depends. + self.aoDepTestCases = []; + + # List of objects of type GlobalResourceData on which current Test Case depends. + self.aoDepGlobalResources = []; + + # List of objects of type TestCaseArgsData. + self.aoTestCaseArgs = []; + + def _initExtraMembersFromDb(self, oDb, tsNow = None, sPeriodBack = None): + """ + Worker shared by the initFromDb* methods. + Returns self. Raises exception if no row or database error. + """ + _ = sPeriodBack; ## @todo sPeriodBack + from testmanager.core.testcaseargs import TestCaseArgsLogic; + self.aoDepTestCases = TestCaseDependencyLogic(oDb).getDepTestCaseData(self.idTestCase, tsNow); + self.aoDepGlobalResources = TestCaseGlobalRsrcDepLogic(oDb).getDepGlobalResourceData(self.idTestCase, tsNow); + self.aoTestCaseArgs = TestCaseArgsLogic(oDb).getTestCaseArgs(self.idTestCase, tsNow); + # Note! The above arrays are sorted by their relvant IDs for fetchForChangeLog's sake. + return self; + + def initFromDbRowEx(self, aoRow, oDb, tsNow = None): + """ + Reinitialize from a SELECT * FROM TestCases row. Will query the + necessary additional data from oDb using tsNow. + Returns self. Raises exception if no row or database error. + """ + TestCaseData.initFromDbRow(self, aoRow); + return self._initExtraMembersFromDb(oDb, tsNow); + + def initFromDbWithId(self, oDb, idTestCase, tsNow = None, sPeriodBack = None): + """ + Initialize the object from the database. + """ + TestCaseData.initFromDbWithId(self, oDb, idTestCase, tsNow, sPeriodBack); + return self._initExtraMembersFromDb(oDb, tsNow, sPeriodBack); + + def initFromDbWithGenId(self, oDb, idGenTestCase, tsNow = None): + """ + Initialize the object from the database. + """ + TestCaseData.initFromDbWithGenId(self, oDb, idGenTestCase); + if tsNow is None and not oDb.isTsInfinity(self.tsExpire): + tsNow = self.tsEffective; + return self._initExtraMembersFromDb(oDb, tsNow); + + def getAttributeParamNullValues(self, sAttr): + if sAttr in ['aoDepTestCases', 'aoDepGlobalResources', 'aoTestCaseArgs']: + return [[], '']; + return TestCaseData.getAttributeParamNullValues(self, sAttr); + + def convertParamToAttribute(self, sAttr, sParam, oValue, oDisp, fStrict): + """For dealing with the arrays.""" + if sAttr not in ['aoDepTestCases', 'aoDepGlobalResources', 'aoTestCaseArgs']: + return TestCaseData.convertParamToAttribute(self, sAttr, sParam, oValue, oDisp, fStrict); + + aoNewValues = []; + if sAttr == 'aoDepTestCases': + for idTestCase in oDisp.getListOfIntParams(sParam, 1, 0x7ffffffe, []): + oDep = TestCaseData(); + oDep.idTestCase = str(idTestCase); + aoNewValues.append(oDep); + + elif sAttr == 'aoDepGlobalResources': + for idGlobalRsrc in oDisp.getListOfIntParams(sParam, 1, 0x7ffffffe, []): + oGlobalRsrc = GlobalResourceData(); + oGlobalRsrc.idGlobalRsrc = str(idGlobalRsrc); + aoNewValues.append(oGlobalRsrc); + + elif sAttr == 'aoTestCaseArgs': + from testmanager.core.testcaseargs import TestCaseArgsData; + for sArgKey in oDisp.getStringParam(TestCaseDataEx.ksParam_aoTestCaseArgs, sDefault = '').split(','): + oDispWrapper = self.DispWrapper(oDisp, '%s[%s][%%s]' % (TestCaseDataEx.ksParam_aoTestCaseArgs, sArgKey,)) + aoNewValues.append(TestCaseArgsData().initFromParams(oDispWrapper, fStrict = False)); + return aoNewValues; + + def _validateAndConvertAttribute(self, sAttr, sParam, oValue, aoNilValues, fAllowNull, oDb): # pylint: disable=too-many-locals + """ + Validate special arrays and requirement expressions. + + For the two dependency arrays we have to supply missing bits by + looking them up in the database. In the argument variation case we + need to validate each item. + """ + if sAttr not in ['aoDepTestCases', 'aoDepGlobalResources', 'aoTestCaseArgs']: + return TestCaseData._validateAndConvertAttribute(self, sAttr, sParam, oValue, aoNilValues, fAllowNull, oDb); + + asErrors = []; + aoNewValues = []; + if sAttr == 'aoDepTestCases': + for oTestCase in self.aoDepTestCases: + if utils.isString(oTestCase.idTestCase): # Stored as string convertParamToAttribute. + oTestCase = copy.copy(oTestCase); + try: + oTestCase.idTestCase = int(oTestCase.idTestCase); + oTestCase.initFromDbWithId(oDb, oTestCase.idTestCase); + except Exception as oXcpt: + asErrors.append('Test case dependency #%s: %s' % (oTestCase.idTestCase, oXcpt)); + aoNewValues.append(oTestCase); + + elif sAttr == 'aoDepGlobalResources': + for oGlobalRsrc in self.aoDepGlobalResources: + if utils.isString(oGlobalRsrc.idGlobalRsrc): # Stored as string convertParamToAttribute. + oGlobalRsrc = copy.copy(oGlobalRsrc); + try: + oGlobalRsrc.idTestCase = int(oGlobalRsrc.idGlobalRsrc); + oGlobalRsrc.initFromDbWithId(oDb, oGlobalRsrc.idGlobalRsrc); + except Exception as oXcpt: + asErrors.append('Resource dependency #%s: %s' % (oGlobalRsrc.idGlobalRsrc, oXcpt)); + aoNewValues.append(oGlobalRsrc); + + else: + assert sAttr == 'aoTestCaseArgs'; + if not self.aoTestCaseArgs: + return (None, 'The testcase requires at least one argument variation to be valid.'); + + # Note! We'll be returning an error dictionary instead of an string here. + dErrors = {}; + + for iVar, oVar in enumerate(self.aoTestCaseArgs): + oVar = copy.copy(oVar); + oVar.idTestCase = self.idTestCase; + dCurErrors = oVar.validateAndConvert(oDb, ModelDataBase.ksValidateFor_Other); + if not dCurErrors: + pass; ## @todo figure out the ID? + else: + asErrors = []; + for sKey in dCurErrors: + asErrors.append('%s: %s' % (sKey[len('TestCaseArgs_'):], dCurErrors[sKey])); + dErrors[iVar] = '<br>\n'.join(asErrors) + aoNewValues.append(oVar); + + for iVar, oVar in enumerate(self.aoTestCaseArgs): + sArgs = oVar.sArgs; + for iVar2 in range(iVar + 1, len(self.aoTestCaseArgs)): + if self.aoTestCaseArgs[iVar2].sArgs == sArgs: + sMsg = 'Duplicate argument variation "%s".' % (sArgs); + if iVar in dErrors: dErrors[iVar] += '<br>\n' + sMsg; + else: dErrors[iVar] = sMsg; + if iVar2 in dErrors: dErrors[iVar2] += '<br>\n' + sMsg; + else: dErrors[iVar2] = sMsg; + break; + + return (aoNewValues, dErrors if dErrors else None); + + return (aoNewValues, None if not asErrors else ' <br>'.join(asErrors)); + + def _validateAndConvertWorker(self, asAllowNullAttributes, oDb, enmValidateFor = ModelDataBase.ksValidateFor_Other): + dErrors = TestCaseData._validateAndConvertWorker(self, asAllowNullAttributes, oDb, enmValidateFor); + + # Validate dependencies a wee bit for paranoid reasons. The scheduler + # queue generation code does the real validation here! + if not dErrors and self.idTestCase is not None: + for oDep in self.aoDepTestCases: + if oDep.idTestCase == self.idTestCase: + if self.ksParam_aoDepTestCases in dErrors: + dErrors[self.ksParam_aoDepTestCases] += ' Depending on itself!'; + else: + dErrors[self.ksParam_aoDepTestCases] = 'Depending on itself!'; + return dErrors; + + + + + +class TestCaseLogic(ModelLogicBase): + """ + Test case management logic. + """ + + def __init__(self, oDb): + ModelLogicBase.__init__(self, oDb) + self.dCache = None; + + def getAll(self): + """ + Fetches all test case records from DB (TestCaseData). + """ + self._oDb.execute('SELECT *\n' + 'FROM TestCases\n' + 'WHERE tsExpire = \'infinity\'::TIMESTAMP\n' + 'ORDER BY idTestCase ASC;') + + aaoRows = self._oDb.fetchAll() + aoRet = []; + for aoRow in aaoRows: + aoRet.append(TestCaseData().initFromDbRow(aoRow)) + return aoRet + + def fetchForListing(self, iStart, cMaxRows, tsNow, aiSortColumns = None): + """ + Fetches test cases. + + Returns an array (list) of TestCaseDataEx items, empty list if none. + Raises exception on error. + """ + _ = aiSortColumns; + if tsNow is None: + self._oDb.execute('SELECT *\n' + 'FROM TestCases\n' + 'WHERE tsExpire = \'infinity\'::TIMESTAMP\n' + 'ORDER BY sName ASC\n' + 'LIMIT %s OFFSET %s\n' + , (cMaxRows, iStart, )); + else: + self._oDb.execute('SELECT *\n' + 'FROM TestCases\n' + 'WHERE tsExpire > %s\n' + ' AND tsEffective <= %s\n' + 'ORDER BY sName ASC\n' + 'LIMIT %s OFFSET %s\n' + , (tsNow, tsNow, cMaxRows, iStart, )); + + aoRows = []; + for aoRow in self._oDb.fetchAll(): + aoRows.append(TestCaseDataEx().initFromDbRowEx(aoRow, self._oDb, tsNow)); + return aoRows; + + def fetchForChangeLog(self, idTestCase, iStart, cMaxRows, tsNow): # pylint: disable=too-many-locals + """ + Fetches change log entries for a testbox. + + Returns an array of ChangeLogEntry instance and an indicator whether + there are more entries. + Raises exception on error. + """ + + if tsNow is None: + tsNow = self._oDb.getCurrentTimestamp(); + + # 1. Get a list of the relevant change times. + self._oDb.execute('( SELECT tsEffective, uidAuthor FROM TestCases WHERE idTestCase = %s AND tsEffective <= %s )\n' + 'UNION\n' + '( SELECT tsEffective, uidAuthor FROM TestCaseArgs WHERE idTestCase = %s AND tsEffective <= %s )\n' + 'UNION\n' + '( SELECT tsEffective, uidAuthor FROM TestCaseDeps WHERE idTestCase = %s AND tsEffective <= %s )\n' + 'UNION\n' + '( SELECT tsEffective, uidAuthor FROM TestCaseGlobalRsrcDeps \n' \ + ' WHERE idTestCase = %s AND tsEffective <= %s )\n' + 'ORDER BY tsEffective DESC\n' + 'LIMIT %s OFFSET %s\n' + , ( idTestCase, tsNow, + idTestCase, tsNow, + idTestCase, tsNow, + idTestCase, tsNow, + cMaxRows + 1, iStart, )); + aaoChanges = self._oDb.fetchAll(); + + # 2. Collect data sets for each of those points. + # (Doing it the lazy + inefficient way for now.) + aoRows = []; + for aoChange in aaoChanges: + aoRows.append(TestCaseDataEx().initFromDbWithId(self._oDb, idTestCase, aoChange[0])); + + # 3. Calculate the changes. + aoEntries = []; + for i in range(0, len(aoRows) - 1): + oNew = aoRows[i]; + oOld = aoRows[i + 1]; + (tsEffective, uidAuthor) = aaoChanges[i]; + (tsExpire, _) = aaoChanges[i - 1] if i > 0 else (oNew.tsExpire, None) + assert self._oDb.isTsInfinity(tsEffective) != self._oDb.isTsInfinity(tsExpire) or tsEffective < tsExpire, \ + '%s vs %s' % (tsEffective, tsExpire); + + aoChanges = []; + + # The testcase object. + if oNew.tsEffective != oOld.tsEffective: + for sAttr in oNew.getDataAttributes(): + if sAttr not in [ 'tsEffective', 'tsExpire', 'uidAuthor', \ + 'aoTestCaseArgs', 'aoDepTestCases', 'aoDepGlobalResources']: + oOldAttr = getattr(oOld, sAttr); + oNewAttr = getattr(oNew, sAttr); + if oOldAttr != oNewAttr: + aoChanges.append(AttributeChangeEntry(sAttr, oNewAttr, oOldAttr, str(oNewAttr), str(oOldAttr))); + + # The argument variations. + iChildOld = 0; + for oChildNew in oNew.aoTestCaseArgs: + # Locate the old entry, emitting removed markers for old items we have to skip. + while iChildOld < len(oOld.aoTestCaseArgs) \ + and oOld.aoTestCaseArgs[iChildOld].idTestCaseArgs < oChildNew.idTestCaseArgs: + oChildOld = oOld.aoTestCaseArgs[iChildOld]; + aoChanges.append(AttributeChangeEntry('Variation #%s' % (oChildOld.idTestCaseArgs,), + None, oChildOld, 'Removed', str(oChildOld))); + iChildOld += 1; + + if iChildOld < len(oOld.aoTestCaseArgs) \ + and oOld.aoTestCaseArgs[iChildOld].idTestCaseArgs == oChildNew.idTestCaseArgs: + oChildOld = oOld.aoTestCaseArgs[iChildOld]; + if oChildNew.tsEffective != oChildOld.tsEffective: + for sAttr in oChildNew.getDataAttributes(): + if sAttr not in [ 'tsEffective', 'tsExpire', 'uidAuthor', 'idGenTestCase', ]: + oOldAttr = getattr(oChildOld, sAttr); + oNewAttr = getattr(oChildNew, sAttr); + if oOldAttr != oNewAttr: + aoChanges.append(AttributeChangeEntry('Variation[#%s].%s' + % (oChildOld.idTestCaseArgs, sAttr,), + oNewAttr, oOldAttr, + str(oNewAttr), str(oOldAttr))); + iChildOld += 1; + else: + aoChanges.append(AttributeChangeEntry('Variation #%s' % (oChildNew.idTestCaseArgs,), + oChildNew, None, + str(oChildNew), 'Did not exist')); + + # The testcase dependencies. + iChildOld = 0; + for oChildNew in oNew.aoDepTestCases: + # Locate the old entry, emitting removed markers for old items we have to skip. + while iChildOld < len(oOld.aoDepTestCases) \ + and oOld.aoDepTestCases[iChildOld].idTestCase < oChildNew.idTestCase: + oChildOld = oOld.aoDepTestCases[iChildOld]; + aoChanges.append(AttributeChangeEntry('Dependency #%s' % (oChildOld.idTestCase,), + None, oChildOld, 'Removed', + '%s (#%u)' % (oChildOld.sName, oChildOld.idTestCase,))); + iChildOld += 1; + if iChildOld < len(oOld.aoDepTestCases) \ + and oOld.aoDepTestCases[iChildOld].idTestCase == oChildNew.idTestCase: + iChildOld += 1; + else: + aoChanges.append(AttributeChangeEntry('Dependency #%s' % (oChildNew.idTestCase,), + oChildNew, None, + '%s (#%u)' % (oChildNew.sName, oChildNew.idTestCase,), + 'Did not exist')); + + # The global resource dependencies. + iChildOld = 0; + for oChildNew in oNew.aoDepGlobalResources: + # Locate the old entry, emitting removed markers for old items we have to skip. + while iChildOld < len(oOld.aoDepGlobalResources) \ + and oOld.aoDepGlobalResources[iChildOld].idGlobalRsrc < oChildNew.idGlobalRsrc: + oChildOld = oOld.aoDepGlobalResources[iChildOld]; + aoChanges.append(AttributeChangeEntry('Global Resource #%s' % (oChildOld.idGlobalRsrc,), + None, oChildOld, 'Removed', + '%s (#%u)' % (oChildOld.sName, oChildOld.idGlobalRsrc,))); + iChildOld += 1; + if iChildOld < len(oOld.aoDepGlobalResources) \ + and oOld.aoDepGlobalResources[iChildOld].idGlobalRsrc == oChildNew.idGlobalRsrc: + iChildOld += 1; + else: + aoChanges.append(AttributeChangeEntry('Global Resource #%s' % (oChildNew.idGlobalRsrc,), + oChildNew, None, + '%s (#%u)' % (oChildNew.sName, oChildNew.idGlobalRsrc,), + 'Did not exist')); + + # Done. + aoEntries.append(ChangeLogEntry(uidAuthor, None, tsEffective, tsExpire, oNew, oOld, aoChanges)); + + # If we're at the end of the log, add the initial entry. + if len(aoRows) <= cMaxRows and aoRows: + oNew = aoRows[-1]; + aoEntries.append(ChangeLogEntry(oNew.uidAuthor, None, + aaoChanges[-1][0], aaoChanges[-2][0] if len(aaoChanges) > 1 else oNew.tsExpire, + oNew, None, [])); + + return (UserAccountLogic(self._oDb).resolveChangeLogAuthors(aoEntries), len(aoRows) > cMaxRows); + + + def addEntry(self, oData, uidAuthor, fCommit = False): + """ + Add a new testcase to the DB. + """ + + # + # Validate the input first. + # + assert isinstance(oData, TestCaseDataEx); + dErrors = oData.validateAndConvert(self._oDb, oData.ksValidateFor_Add); + if dErrors: + raise TMInvalidData('Invalid input data: %s' % (dErrors,)); + + # + # Add the testcase. + # + self._oDb.callProc('TestCaseLogic_addEntry', + ( uidAuthor, oData.sName, oData.sDescription, oData.fEnabled, oData.cSecTimeout, + oData.sTestBoxReqExpr, oData.sBuildReqExpr, oData.sBaseCmd, oData.sValidationKitZips, + oData.sComment )); + oData.idTestCase = self._oDb.fetchOne()[0]; + + # Add testcase dependencies. + for oDep in oData.aoDepTestCases: + self._oDb.execute('INSERT INTO TestCaseDeps (idTestCase, idTestCasePreReq, uidAuthor) VALUES (%s, %s, %s)' + , (oData.idTestCase, oDep.idTestCase, uidAuthor)) + + # Add global resource dependencies. + for oDep in oData.aoDepGlobalResources: + self._oDb.execute('INSERT INTO TestCaseGlobalRsrcDeps (idTestCase, idGlobalRsrc, uidAuthor) VALUES (%s, %s, %s)' + , (oData.idTestCase, oDep.idGlobalRsrc, uidAuthor)) + + # Set Test Case Arguments variations + for oVar in oData.aoTestCaseArgs: + self._oDb.execute('INSERT INTO TestCaseArgs (\n' + ' idTestCase, uidAuthor, sArgs, cSecTimeout,\n' + ' sTestBoxReqExpr, sBuildReqExpr, cGangMembers, sSubName)\n' + 'VALUES (%s, %s, %s, %s, %s, %s, %s, %s)' + , ( oData.idTestCase, uidAuthor, oVar.sArgs, oVar.cSecTimeout, + oVar.sTestBoxReqExpr, oVar.sBuildReqExpr, oVar.cGangMembers, oVar.sSubName, )); + + self._oDb.maybeCommit(fCommit); + return True; + + def editEntry(self, oData, uidAuthor, fCommit = False): # pylint: disable=too-many-locals + """ + Edit a testcase entry (extended). + Caller is expected to rollback the database transactions on exception. + """ + + # + # Validate the input. + # + assert isinstance(oData, TestCaseDataEx); + dErrors = oData.validateAndConvert(self._oDb, oData.ksValidateFor_Edit); + if dErrors: + raise TMInvalidData('Invalid input data: %s' % (dErrors,)); + + # + # Did anything change? If not return straight away. + # + oOldDataEx = TestCaseDataEx().initFromDbWithId(self._oDb, oData.idTestCase); + if oOldDataEx.isEqual(oData): + self._oDb.maybeCommit(fCommit); + return True; + + # + # Make the necessary changes. + # + + # The test case itself. + if not TestCaseData().initFromOther(oOldDataEx).isEqual(oData): + self._oDb.callProc('TestCaseLogic_editEntry', ( uidAuthor, oData.idTestCase, oData.sName, oData.sDescription, + oData.fEnabled, oData.cSecTimeout, oData.sTestBoxReqExpr, + oData.sBuildReqExpr, oData.sBaseCmd, oData.sValidationKitZips, + oData.sComment )); + oData.idGenTestCase = self._oDb.fetchOne()[0]; + + # + # Its dependencies on other testcases. + # + aidNewDeps = [oDep.idTestCase for oDep in oData.aoDepTestCases]; + aidOldDeps = [oDep.idTestCase for oDep in oOldDataEx.aoDepTestCases]; + + sQuery = self._oDb.formatBindArgs('UPDATE TestCaseDeps\n' + 'SET tsExpire = CURRENT_TIMESTAMP\n' + 'WHERE idTestCase = %s\n' + ' AND tsExpire = \'infinity\'::timestamp\n' + , (oData.idTestCase,)); + asKeepers = []; + for idDep in aidOldDeps: + if idDep in aidNewDeps: + asKeepers.append(str(idDep)); + if asKeepers: + sQuery += ' AND idTestCasePreReq NOT IN (' + ', '.join(asKeepers) + ')\n'; + self._oDb.execute(sQuery); + + for idDep in aidNewDeps: + if idDep not in aidOldDeps: + self._oDb.execute('INSERT INTO TestCaseDeps (idTestCase, idTestCasePreReq, uidAuthor)\n' + 'VALUES (%s, %s, %s)\n' + , (oData.idTestCase, idDep, uidAuthor) ); + + # + # Its dependencies on global resources. + # + aidNewDeps = [oDep.idGlobalRsrc for oDep in oData.aoDepGlobalResources]; + aidOldDeps = [oDep.idGlobalRsrc for oDep in oOldDataEx.aoDepGlobalResources]; + + sQuery = self._oDb.formatBindArgs('UPDATE TestCaseGlobalRsrcDeps\n' + 'SET tsExpire = CURRENT_TIMESTAMP\n' + 'WHERE idTestCase = %s\n' + ' AND tsExpire = \'infinity\'::timestamp\n' + , (oData.idTestCase,)); + asKeepers = []; + for idDep in aidOldDeps: + if idDep in aidNewDeps: + asKeepers.append(str(idDep)); + if asKeepers: + sQuery = ' AND idGlobalRsrc NOT IN (' + ', '.join(asKeepers) + ')\n'; + self._oDb.execute(sQuery); + + for idDep in aidNewDeps: + if idDep not in aidOldDeps: + self._oDb.execute('INSERT INTO TestCaseGlobalRsrcDeps (idTestCase, idGlobalRsrc, uidAuthor)\n' + 'VALUES (%s, %s, %s)\n' + , (oData.idTestCase, idDep, uidAuthor) ); + + # + # Update Test Case Args + # Note! Primary key is idTestCase, tsExpire, sArgs. + # + + # Historize rows that have been removed. + sQuery = self._oDb.formatBindArgs('UPDATE TestCaseArgs\n' + 'SET tsExpire = CURRENT_TIMESTAMP\n' + 'WHERE idTestCase = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP' + , (oData.idTestCase, )); + for oNewVar in oData.aoTestCaseArgs: + asKeepers.append(self._oDb.formatBindArgs('%s', (oNewVar.sArgs,))); + if asKeepers: + sQuery += ' AND sArgs NOT IN (' + ', '.join(asKeepers) + ')\n'; + self._oDb.execute(sQuery); + + # Add new TestCaseArgs records if necessary, reusing old IDs when possible. + from testmanager.core.testcaseargs import TestCaseArgsData; + for oNewVar in oData.aoTestCaseArgs: + self._oDb.execute('SELECT *\n' + 'FROM TestCaseArgs\n' + 'WHERE idTestCase = %s\n' + ' AND sArgs = %s\n' + 'ORDER BY tsExpire DESC\n' + 'LIMIT 1\n' + , (oData.idTestCase, oNewVar.sArgs,)); + aoRow = self._oDb.fetchOne(); + if aoRow is None: + # New + self._oDb.execute('INSERT INTO TestCaseArgs (\n' + ' idTestCase, uidAuthor, sArgs, cSecTimeout,\n' + ' sTestBoxReqExpr, sBuildReqExpr, cGangMembers, sSubName)\n' + 'VALUES (%s, %s, %s, %s, %s, %s, %s, %s)' + , ( oData.idTestCase, uidAuthor, oNewVar.sArgs, oNewVar.cSecTimeout, + oNewVar.sTestBoxReqExpr, oNewVar.sBuildReqExpr, oNewVar.cGangMembers, oNewVar.sSubName)); + else: + oCurVar = TestCaseArgsData().initFromDbRow(aoRow); + if self._oDb.isTsInfinity(oCurVar.tsExpire): + # Existing current entry, updated if changed. + if oNewVar.cSecTimeout == oCurVar.cSecTimeout \ + and oNewVar.sTestBoxReqExpr == oCurVar.sTestBoxReqExpr \ + and oNewVar.sBuildReqExpr == oCurVar.sBuildReqExpr \ + and oNewVar.cGangMembers == oCurVar.cGangMembers \ + and oNewVar.sSubName == oCurVar.sSubName: + oNewVar.idTestCaseArgs = oCurVar.idTestCaseArgs; + oNewVar.idGenTestCaseArgs = oCurVar.idGenTestCaseArgs; + continue; # Unchanged. + self._oDb.execute('UPDATE TestCaseArgs SET tsExpire = CURRENT_TIMESTAMP WHERE idGenTestCaseArgs = %s\n' + , (oCurVar.idGenTestCaseArgs, )); + else: + # Existing old entry, re-use the ID. + pass; + self._oDb.execute('INSERT INTO TestCaseArgs (\n' + ' idTestCaseArgs, idTestCase, uidAuthor, sArgs, cSecTimeout,\n' + ' sTestBoxReqExpr, sBuildReqExpr, cGangMembers, sSubName)\n' + 'VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)\n' + 'RETURNING idGenTestCaseArgs\n' + , ( oCurVar.idTestCaseArgs, oData.idTestCase, uidAuthor, oNewVar.sArgs, oNewVar.cSecTimeout, + oNewVar.sTestBoxReqExpr, oNewVar.sBuildReqExpr, oNewVar.cGangMembers, oNewVar.sSubName)); + oNewVar.idGenTestCaseArgs = self._oDb.fetchOne()[0]; + + self._oDb.maybeCommit(fCommit); + return True; + + def removeEntry(self, uidAuthor, idTestCase, fCascade = False, fCommit = False): + """ Deletes the test case if possible. """ + self._oDb.callProc('TestCaseLogic_delEntry', (uidAuthor, idTestCase, fCascade)); + self._oDb.maybeCommit(fCommit); + return True + + + def getTestCasePreReqIds(self, idTestCase, tsEffective = None, cMax = None): + """ + Returns an array of prerequisite testcases (IDs) for the given testcase. + May raise exception on database error or if the result exceeds cMax. + """ + if tsEffective is None: + self._oDb.execute('SELECT idTestCasePreReq\n' + 'FROM TestCaseDeps\n' + 'WHERE idTestCase = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + 'ORDER BY idTestCasePreReq\n' + , (idTestCase,) ); + else: + self._oDb.execute('SELECT idTestCasePreReq\n' + 'FROM TestCaseDeps\n' + 'WHERE idTestCase = %s\n' + ' AND tsExpire > %s\n' + ' AND tsEffective <= %s\n' + 'ORDER BY idTestCasePreReq\n' + , (idTestCase, tsEffective, tsEffective) ); + + + if cMax is not None and self._oDb.getRowCount() > cMax: + raise TMExceptionBase('Too many prerequisites for testcase %s: %s, max %s' + % (idTestCase, cMax, self._oDb.getRowCount(),)); + + aidPreReqs = []; + for aoRow in self._oDb.fetchAll(): + aidPreReqs.append(aoRow[0]); + return aidPreReqs; + + + def cachedLookup(self, idTestCase): + """ + Looks up the most recent TestCaseDataEx object for idTestCase + via an object cache. + + Returns a shared TestCaseDataEx object. None if not found. + Raises exception on DB error. + """ + if self.dCache is None: + self.dCache = self._oDb.getCache('TestCaseDataEx'); + oEntry = self.dCache.get(idTestCase, None); + if oEntry is None: + fNeedTsNow = False; + self._oDb.execute('SELECT *\n' + 'FROM TestCases\n' + 'WHERE idTestCase = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , (idTestCase, )); + if self._oDb.getRowCount() == 0: + # Maybe it was deleted, try get the last entry. + self._oDb.execute('SELECT *\n' + 'FROM TestCases\n' + 'WHERE idTestCase = %s\n' + 'ORDER BY tsExpire DESC\n' + 'LIMIT 1\n' + , (idTestCase, )); + fNeedTsNow = True; + elif self._oDb.getRowCount() > 1: + raise self._oDb.integrityException('%s infinity rows for %s' % (self._oDb.getRowCount(), idTestCase)); + + if self._oDb.getRowCount() == 1: + aaoRow = self._oDb.fetchOne(); + oEntry = TestCaseDataEx(); + tsNow = oEntry.initFromDbRow(aaoRow).tsEffective if fNeedTsNow else None; + oEntry.initFromDbRowEx(aaoRow, self._oDb, tsNow); + self.dCache[idTestCase] = oEntry; + return oEntry; + + + +# +# Unit testing. +# + +# pylint: disable=missing-docstring +class TestCaseGlobalRsrcDepDataTestCase(ModelDataBaseTestCase): + def setUp(self): + self.aoSamples = [TestCaseGlobalRsrcDepData(),]; + +class TestCaseDataTestCase(ModelDataBaseTestCase): + def setUp(self): + self.aoSamples = [TestCaseData(),]; + + def testEmptyExpr(self): + self.assertEqual(TestCaseData.validateTestBoxReqExpr(None), None); + self.assertEqual(TestCaseData.validateTestBoxReqExpr(''), None); + + def testSimpleExpr(self): + self.assertEqual(TestCaseData.validateTestBoxReqExpr('cMbMemory > 10'), None); + self.assertEqual(TestCaseData.validateTestBoxReqExpr('cMbScratch < 10'), None); + self.assertEqual(TestCaseData.validateTestBoxReqExpr('fChipsetIoMmu'), None); + self.assertEqual(TestCaseData.validateTestBoxReqExpr('fChipsetIoMmu is True'), None); + self.assertEqual(TestCaseData.validateTestBoxReqExpr('fChipsetIoMmu is False'), None); + self.assertEqual(TestCaseData.validateTestBoxReqExpr('fChipsetIoMmu is None'), None); + self.assertEqual(TestCaseData.validateTestBoxReqExpr('isinstance(fChipsetIoMmu, bool)'), None); + self.assertEqual(TestCaseData.validateTestBoxReqExpr('isinstance(iTestBoxScriptRev, int)'), None); + self.assertEqual(TestCaseData.validateTestBoxReqExpr('isinstance(cMbScratch, long)'), None); + + def testBadExpr(self): + self.assertNotEqual(TestCaseData.validateTestBoxReqExpr('this is an bad expression, surely it must be'), None); + self.assertNotEqual(TestCaseData.validateTestBoxReqExpr('x = 1 + 1'), None); + self.assertNotEqual(TestCaseData.validateTestBoxReqExpr('__import__(\'os\').unlink(\'/tmp/no/such/file\')'), None); + self.assertNotEqual(TestCaseData.validateTestBoxReqExpr('print "foobar"'), None); + +class TestCaseDataExTestCase(ModelDataBaseTestCase): + def setUp(self): + self.aoSamples = [TestCaseDataEx(),]; + +if __name__ == '__main__': + unittest.main(); + # not reached. + diff --git a/src/VBox/ValidationKit/testmanager/core/testcaseargs.py b/src/VBox/ValidationKit/testmanager/core/testcaseargs.py new file mode 100755 index 00000000..e63df090 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/testcaseargs.py @@ -0,0 +1,416 @@ +# -*- coding: utf-8 -*- +# $Id: testcaseargs.py $ + +""" +Test Manager - Test Case Arguments Variations. +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Standard python imports. +import unittest; +import sys; + +# Validation Kit imports. +from common import utils; +from testmanager.core.base import ModelDataBase, ModelDataBaseTestCase, ModelLogicBase, TMExceptionBase, \ + TMRowNotFound; +from testmanager.core.testcase import TestCaseData, TestCaseDependencyLogic, TestCaseGlobalRsrcDepLogic; + +# Python 3 hacks: +if sys.version_info[0] >= 3: + long = int; # pylint: disable=redefined-builtin,invalid-name + + +class TestCaseArgsData(ModelDataBase): + """ + Test case argument variation. + """ + + ksIdAttr = 'idTestCaseArgs'; + ksIdGenAttr = 'idGenTestCaseArgs'; + + ksParam_idTestCase = 'TestCaseArgs_idTestCase'; + ksParam_idTestCaseArgs = 'TestCaseArgs_idTestCaseArgs'; + ksParam_tsEffective = 'TestCaseArgs_tsEffective'; + ksParam_tsExpire = 'TestCaseArgs_tsExpire'; + ksParam_uidAuthor = 'TestCaseArgs_uidAuthor'; + ksParam_idGenTestCaseArgs = 'TestCaseArgs_idGenTestCaseArgs'; + ksParam_sArgs = 'TestCaseArgs_sArgs'; + ksParam_cSecTimeout = 'TestCaseArgs_cSecTimeout'; + ksParam_sTestBoxReqExpr = 'TestCaseArgs_sTestBoxReqExpr'; + ksParam_sBuildReqExpr = 'TestCaseArgs_sBuildReqExpr'; + ksParam_cGangMembers = 'TestCaseArgs_cGangMembers'; + ksParam_sSubName = 'TestCaseArgs_sSubName'; + + kcDbColumns = 12; + + kasAllowNullAttributes = [ 'idTestCase', 'idTestCaseArgs', 'tsEffective', 'tsExpire', 'uidAuthor', 'idGenTestCaseArgs', + 'cSecTimeout', 'sTestBoxReqExpr', 'sBuildReqExpr', 'sSubName', ]; + + def __init__(self): + ModelDataBase.__init__(self); + + # + # Initialize with defaults. + # See the database for explanations of each of these fields. + # + self.idTestCase = None; + self.idTestCaseArgs = None; + self.tsEffective = None; + self.tsExpire = None; + self.uidAuthor = None; + self.idGenTestCaseArgs = None; + self.sArgs = ''; + self.cSecTimeout = None; + self.sTestBoxReqExpr = None; + self.sBuildReqExpr = None; + self.cGangMembers = 1; + self.sSubName = None; + + def initFromDbRow(self, aoRow): + """ + Re-initializes the object from a SELECT * FROM TestCaseArgs row. + Returns self. Raises exception if aoRow is None. + """ + if aoRow is None: + raise TMRowNotFound('TestBoxStatus not found.'); + + self.idTestCase = aoRow[0]; + self.idTestCaseArgs = aoRow[1]; + self.tsEffective = aoRow[2]; + self.tsExpire = aoRow[3]; + self.uidAuthor = aoRow[4]; + self.idGenTestCaseArgs = aoRow[5]; + self.sArgs = aoRow[6]; + self.cSecTimeout = aoRow[7]; + self.sTestBoxReqExpr = aoRow[8]; + self.sBuildReqExpr = aoRow[9]; + self.cGangMembers = aoRow[10]; + self.sSubName = aoRow[11]; + return self; + + def initFromDbWithId(self, oDb, idTestCaseArgs, tsNow = None, sPeriodBack = None): + """ + Initialize from the database. + """ + oDb.execute(self.formatSimpleNowAndPeriodQuery(oDb, + 'SELECT *\n' + 'FROM TestCaseArgs\n' + 'WHERE idTestCaseArgs = %s\n' + , ( idTestCaseArgs,), tsNow, sPeriodBack)); + aoRow = oDb.fetchOne() + if aoRow is None: + raise TMRowNotFound('idTestCaseArgs=%s not found (tsNow=%s sPeriodBack=%s)' + % (idTestCaseArgs, tsNow, sPeriodBack,)); + return self.initFromDbRow(aoRow); + + def initFromDbWithGenId(self, oDb, idGenTestCaseArgs): + """ + Initialize from the database, given the generation ID of a row. + """ + oDb.execute('SELECT * FROM TestCaseArgs WHERE idGenTestCaseArgs = %s', (idGenTestCaseArgs,)); + return self.initFromDbRow(oDb.fetchOne()); + + def initFromValues(self, sArgs, cSecTimeout = None, sTestBoxReqExpr = None, sBuildReqExpr = None, # pylint: disable=too-many-arguments + cGangMembers = 1, idTestCase = None, idTestCaseArgs = None, tsEffective = None, tsExpire = None, + uidAuthor = None, idGenTestCaseArgs = None, sSubName = None): + """ + Reinitialize from values. + Returns self. + """ + self.idTestCase = idTestCase; + self.idTestCaseArgs = idTestCaseArgs; + self.tsEffective = tsEffective; + self.tsExpire = tsExpire; + self.uidAuthor = uidAuthor; + self.idGenTestCaseArgs = idGenTestCaseArgs; + self.sArgs = sArgs; + self.cSecTimeout = utils.parseIntervalSeconds(cSecTimeout); + self.sTestBoxReqExpr = sTestBoxReqExpr; + self.sBuildReqExpr = sBuildReqExpr; + self.cGangMembers = cGangMembers; + self.sSubName = sSubName; + return self; + + def getAttributeParamNullValues(self, sAttr): + aoNilValues = ModelDataBase.getAttributeParamNullValues(self, sAttr); + if sAttr == 'cSecTimeout': + aoNilValues.insert(0, ''); # Prettier NULL value for cSecTimeout. + elif sAttr == 'sArgs': + aoNilValues = []; # No NULL value here, thank you. + return aoNilValues; + + def _validateAndConvertAttribute(self, sAttr, sParam, oValue, aoNilValues, fAllowNull, oDb): + if sAttr == 'cSecTimeout' and oValue not in aoNilValues: # Allow human readable interval formats. + return utils.parseIntervalSeconds(oValue); + + (oValue, sError) = ModelDataBase._validateAndConvertAttribute(self, sAttr, sParam, oValue, aoNilValues, fAllowNull, oDb); + if sError is None: + if sAttr == 'sTestBoxReqExpr': + sError = TestCaseData.validateTestBoxReqExpr(oValue); + elif sAttr == 'sBuildReqExpr': + sError = TestCaseData.validateBuildReqExpr(oValue); + return (oValue, sError); + + + + +class TestCaseArgsDataEx(TestCaseArgsData): + """ + Complete data set. + """ + + def __init__(self): + TestCaseArgsData.__init__(self); + self.oTestCase = None; + self.aoTestCasePreReqs = []; + self.aoGlobalRsrc = []; + + def initFromDbRow(self, aoRow): + raise TMExceptionBase('Do not call me: %s' % (aoRow,)) + + def initFromDbRowEx(self, aoRow, oDb, tsConfigEff = None, tsRsrcEff = None): + """ + Extended version of initFromDbRow that fills in the rest from the database. + """ + TestCaseArgsData.initFromDbRow(self, aoRow); + + if tsConfigEff is None: tsConfigEff = oDb.getCurrentTimestamp(); + if tsRsrcEff is None: tsRsrcEff = oDb.getCurrentTimestamp(); + + self.oTestCase = TestCaseData().initFromDbWithId(oDb, self.idTestCase, tsConfigEff); + self.aoTestCasePreReqs = TestCaseDependencyLogic(oDb).getTestCaseDeps(self.idTestCase, tsConfigEff); + self.aoGlobalRsrc = TestCaseGlobalRsrcDepLogic(oDb).getTestCaseDeps(self.idTestCase, tsRsrcEff); + + return self; + + def initFromDbWithId(self, oDb, idTestCaseArgs, tsNow = None, sPeriodBack = None): + _ = oDb; _ = idTestCaseArgs; _ = tsNow; _ = sPeriodBack; + raise TMExceptionBase('Not supported.'); + + def initFromDbWithGenId(self, oDb, idGenTestCaseArgs): + _ = oDb; _ = idGenTestCaseArgs; + raise TMExceptionBase('Use initFromDbWithGenIdEx...'); + + def initFromDbWithGenIdEx(self, oDb, idGenTestCaseArgs, tsConfigEff = None, tsRsrcEff = None): + """ + Initialize from the database, given the ID of a row. + """ + oDb.execute('SELECT *, CURRENT_TIMESTAMP FROM TestCaseArgs WHERE idGenTestCaseArgs = %s', (idGenTestCaseArgs,)); + aoRow = oDb.fetchOne(); + return self.initFromDbRowEx(aoRow, oDb, tsConfigEff, tsRsrcEff); + + def convertFromParamNull(self): + raise TMExceptionBase('Not implemented'); + + def convertToParamNull(self): + raise TMExceptionBase('Not implemented'); + + def isEqual(self, oOther): + raise TMExceptionBase('Not implemented'); + + def matchesTestBoxProps(self, oTestBoxData): + """ + Checks if the all of the testbox related test requirements matches the + given testbox. + + Returns True or False according to the expression, None on exception or + non-boolean expression result. + """ + return TestCaseData.matchesTestBoxPropsEx(oTestBoxData, self.oTestCase.sTestBoxReqExpr) \ + and TestCaseData.matchesTestBoxPropsEx(oTestBoxData, self.sTestBoxReqExpr); + + def matchesBuildProps(self, oBuildDataEx): + """ + Checks if the all of the build related test requirements matches the + given build. + + Returns True or False according to the expression, None on exception or + non-boolean expression result. + """ + return TestCaseData.matchesBuildPropsEx(oBuildDataEx, self.oTestCase.sBuildReqExpr) \ + and TestCaseData.matchesBuildPropsEx(oBuildDataEx, self.sBuildReqExpr); + + +class TestCaseArgsLogic(ModelLogicBase): + """ + TestCaseArgs database logic. + """ + + def __init__(self, oDb): + ModelLogicBase.__init__(self, oDb); + self.dCache = None; + + + def areResourcesFree(self, oDataEx): + """ + Checks if all global resources are currently still in existance and free. + Returns True/False. May raise exception on database error. + """ + + # Create a set of global resource IDs. + if not oDataEx.aoGlobalRsrc: + return True; + asIdRsrcs = [str(oDep.idGlobalRsrc) for oDep, _ in oDataEx.aoGlobalRsrc]; + + # A record in the resource status table means it's allocated. + self._oDb.execute('SELECT COUNT(*)\n' + 'FROM GlobalResourceStatuses\n' + 'WHERE GlobalResourceStatuses.idGlobalRsrc IN (' + ', '.join(asIdRsrcs) + ')\n'); + if self._oDb.fetchOne()[0] == 0: + # Check for disabled or deleted resources (we cannot allocate them). + self._oDb.execute('SELECT COUNT(*)\n' + 'FROM GlobalResources\n' + 'WHERE GlobalResources.idGlobalRsrc IN (' + ', '.join(asIdRsrcs) + ')\n' + ' AND GlobalResources.tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND GlobalResources.fEnabled = TRUE\n'); + if self._oDb.fetchOne()[0] == len(oDataEx.aoGlobalRsrc): + return True; + return False; + + def getAll(self): + """Get list of objects of type TestCaseArgsData""" + self._oDb.execute('SELECT *\n' + 'FROM TestCaseArgs\n' + 'WHERE tsExpire = \'infinity\'::TIMESTAMP') + aaoRows = self._oDb.fetchAll() + aoRet = [] + for aoRow in aaoRows: + aoRet.append(TestCaseArgsData().initFromDbRow(aoRow)) + + return aoRet + + def getTestCaseArgs(self, idTestCase, tsNow = None, aiWhiteList = None): + """Get list of testcase's arguments variations""" + if aiWhiteList is None: + if tsNow is None: + self._oDb.execute('SELECT *\n' + 'FROM TestCaseArgs\n' + 'WHERE idTestCase = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + 'ORDER BY TestCaseArgs.idTestCaseArgs\n' + , (idTestCase,)); + else: + self._oDb.execute('SELECT *\n' + 'FROM TestCaseArgs\n' + 'WHERE idTestCase = %s\n' + ' AND tsExpire > %s\n' + ' AND tsEffective <= %s\n' + 'ORDER BY TestCaseArgs.idTestCaseArgs\n' + , (idTestCase, tsNow, tsNow)); + else: + sWhiteList = ','.join((str(x) for x in aiWhiteList)); + if tsNow is None: + self._oDb.execute('SELECT *\n' + 'FROM TestCaseArgs\n' + 'WHERE idTestCase = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND idTestCaseArgs IN (' + sWhiteList + ')\n' + 'ORDER BY TestCaseArgs.idTestCaseArgs\n' + , (idTestCase,)); + else: + self._oDb.execute('SELECT *\n' + 'FROM TestCaseArgs\n' + 'WHERE idTestCase = %s\n' + ' AND tsExpire > %s\n' + ' AND tsEffective <= %s\n' + ' AND idTestCaseArgs IN (' + sWhiteList + ')\n' + 'ORDER BY TestCaseArgs.idTestCaseArgs\n' + , (idTestCase, tsNow, tsNow)); + + aaoRows = self._oDb.fetchAll() + aoRet = [] + for aoRow in aaoRows: + aoRet.append(TestCaseArgsData().initFromDbRow(aoRow)) + + return aoRet + + def addTestCaseArgs(self, oTestCaseArgsData): + """Add Test Case Args record into DB""" + pass; # pylint: disable=unnecessary-pass + + def cachedLookup(self, idTestCaseArgs): + """ + Looks up the most recent TestCaseArgsDataEx object for idTestCaseArg + via in an object cache. + + Returns a shared TestCaseArgDataEx object. None if not found. + Raises exception on DB error. + """ + if self.dCache is None: + self.dCache = self._oDb.getCache('TestCaseArgsDataEx'); + oEntry = self.dCache.get(idTestCaseArgs, None); + if oEntry is None: + fNeedTsNow = False; + self._oDb.execute('SELECT *\n' + 'FROM TestCaseArgs\n' + 'WHERE idTestCaseArgs = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , (idTestCaseArgs, )); + if self._oDb.getRowCount() == 0: + # Maybe it was deleted, try get the last entry. + self._oDb.execute('SELECT *\n' + 'FROM TestCaseArgs\n' + 'WHERE idTestCaseArgs = %s\n' + 'ORDER BY tsExpire DESC\n' + 'LIMIT 1\n' + , (idTestCaseArgs, )); + fNeedTsNow = True; + elif self._oDb.getRowCount() > 1: + raise self._oDb.integrityException('%s infinity rows for %s' % (self._oDb.getRowCount(), idTestCaseArgs)); + + if self._oDb.getRowCount() == 1: + aaoRow = self._oDb.fetchOne(); + oEntry = TestCaseArgsDataEx(); + tsNow = TestCaseArgsData().initFromDbRow(aaoRow).tsEffective if fNeedTsNow else None; + oEntry.initFromDbRowEx(aaoRow, self._oDb, tsNow, tsNow); + self.dCache[idTestCaseArgs] = oEntry; + return oEntry; + + +# +# Unit testing. +# + +# pylint: disable=missing-docstring +class TestCaseArgsDataTestCase(ModelDataBaseTestCase): + def setUp(self): + self.aoSamples = [TestCaseArgsData(),]; + +if __name__ == '__main__': + unittest.main(); + # not reached. + diff --git a/src/VBox/ValidationKit/testmanager/core/testgroup.py b/src/VBox/ValidationKit/testmanager/core/testgroup.py new file mode 100755 index 00000000..9a648b0d --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/testgroup.py @@ -0,0 +1,771 @@ +# -*- coding: utf-8 -*- +# $Id: testgroup.py $ + +""" +Test Manager - Test groups management. +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Standard python imports. +import unittest; + +# Validation Kit imports. +from testmanager.core.base import ModelDataBase, ModelDataBaseTestCase, ModelLogicBase, TMRowInUse, \ + TMTooManyRows, TMInvalidData, TMRowNotFound, TMRowAlreadyExists; +from testmanager.core.testcase import TestCaseData, TestCaseDataEx; + + +class TestGroupMemberData(ModelDataBase): + """Representation of a test group member database row.""" + + ksParam_idTestGroup = 'TestGroupMember_idTestGroup'; + ksParam_idTestCase = 'TestGroupMember_idTestCase'; + ksParam_tsEffective = 'TestGroupMember_tsEffective'; + ksParam_tsExpire = 'TestGroupMember_tsExpire'; + ksParam_uidAuthor = 'TestGroupMember_uidAuthor'; + ksParam_iSchedPriority = 'TestGroupMember_iSchedPriority'; + ksParam_aidTestCaseArgs = 'TestGroupMember_aidTestCaseArgs'; + + kasAllowNullAttributes = ['idTestGroup', 'idTestCase', 'tsEffective', 'tsExpire', 'uidAuthor', 'aidTestCaseArgs' ]; + kiMin_iSchedPriority = 0; + kiMax_iSchedPriority = 31; + + kcDbColumns = 7; + + def __init__(self): + ModelDataBase.__init__(self) + + # + # Initialize with defaults. + # See the database for explanations of each of these fields. + # + self.idTestGroup = None; + self.idTestCase = None; + self.tsEffective = None; + self.tsExpire = None; + self.uidAuthor = None; + self.iSchedPriority = 16; + self.aidTestCaseArgs = None; + + def initFromDbRow(self, aoRow): + """ + Reinitialize from a SELECT * FROM TestCaseGroupMembers. + Return self. Raises exception if no row. + """ + if aoRow is None: + raise TMRowNotFound('Test group member not found.') + + self.idTestGroup = aoRow[0]; + self.idTestCase = aoRow[1]; + self.tsEffective = aoRow[2]; + self.tsExpire = aoRow[3]; + self.uidAuthor = aoRow[4]; + self.iSchedPriority = aoRow[5]; + self.aidTestCaseArgs = aoRow[6]; + return self + + + def getAttributeParamNullValues(self, sAttr): + # Arrays default to [] as NULL currently. That doesn't work for us. + if sAttr == 'aidTestCaseArgs': + aoNilValues = [None, '-1']; + else: + aoNilValues = ModelDataBase.getAttributeParamNullValues(self, sAttr); + return aoNilValues; + + def _validateAndConvertAttribute(self, sAttr, sParam, oValue, aoNilValues, fAllowNull, oDb): + if sAttr != 'aidTestCaseArgs': + return ModelDataBase._validateAndConvertAttribute(self, sAttr, sParam, oValue, aoNilValues, fAllowNull, oDb); + + # -1 is a special value, which when present make the whole thing NULL (None). + (aidVariations, sError) = self.validateListOfInts(oValue, aoNilValues = aoNilValues, fAllowNull = fAllowNull, + iMin = -1, iMax = 0x7ffffffe); + if sError is None: + if aidVariations is None: + pass; + elif -1 in aidVariations: + aidVariations = None; + elif 0 in aidVariations: + sError = 'Invalid test case varation ID #0.'; + else: + aidVariations = sorted(aidVariations); + return (aidVariations, sError); + + + +class TestGroupMemberDataEx(TestGroupMemberData): + """Extended representation of a test group member.""" + + def __init__(self): + """Extend parent class""" + TestGroupMemberData.__init__(self) + self.oTestCase = None; # TestCaseDataEx. + + def initFromDbRowEx(self, aoRow, oDb, tsNow = None): + """ + Reinitialize from a SELECT * FROM TestGroupMembers, TestCases row. + Will query the necessary additional data from oDb using tsNow. + + Returns self. Raises exception if no row or database error. + """ + TestGroupMemberData.initFromDbRow(self, aoRow); + self.oTestCase = TestCaseDataEx(); + self.oTestCase.initFromDbRowEx(aoRow[TestGroupMemberData.kcDbColumns:], oDb, tsNow); + return self; + + def initFromParams(self, oDisp, fStrict = True): + self.oTestCase = None; + return TestGroupMemberData.initFromParams(self, oDisp, fStrict); + + def getDataAttributes(self): + asAttributes = TestGroupMemberData.getDataAttributes(self); + asAttributes.remove('oTestCase'); + return asAttributes; + + def _validateAndConvertWorker(self, asAllowNullAttributes, oDb, enmValidateFor = ModelDataBase.ksValidateFor_Other): + dErrors = TestGroupMemberData._validateAndConvertWorker(self, asAllowNullAttributes, oDb, enmValidateFor); + if self.ksParam_idTestCase not in dErrors: + self.oTestCase = TestCaseDataEx() + try: + self.oTestCase.initFromDbWithId(oDb, self.idTestCase); + except Exception as oXcpt: + self.oTestCase = TestCaseDataEx() + dErrors[self.ksParam_idTestCase] = str(oXcpt); + return dErrors; + + +class TestGroupMemberData2(TestCaseData): + """Special representation of a Test Group Member item""" + + def __init__(self): + """Extend parent class""" + TestCaseData.__init__(self) + self.idTestGroup = None + self.aidTestCaseArgs = [] + + def initFromDbRowEx(self, aoRow): + """ + Reinitialize from this query: + + SELECT TestCases.*, + TestGroupMembers.idTestGroup, + TestGroupMembers.aidTestCaseArgs + FROM TestCases, TestGroupMembers + WHERE TestCases.idTestCase = TestGroupMembers.idTestCase + + Represents complete test group member (test case) info. + Returns object of type TestGroupMemberData2. Raises exception if no row. + """ + TestCaseData.initFromDbRow(self, aoRow); + self.idTestGroup = aoRow[-2] + self.aidTestCaseArgs = aoRow[-1] + return self; + + +class TestGroupData(ModelDataBase): + """ + Test group data. + """ + + ksIdAttr = 'idTestGroup'; + + ksParam_idTestGroup = 'TestGroup_idTestGroup' + ksParam_tsEffective = 'TestGroup_tsEffective' + ksParam_tsExpire = 'TestGroup_tsExpire' + ksParam_uidAuthor = 'TestGroup_uidAuthor' + ksParam_sName = 'TestGroup_sName' + ksParam_sDescription = 'TestGroup_sDescription' + ksParam_sComment = 'TestGroup_sComment' + + kasAllowNullAttributes = ['idTestGroup', 'tsEffective', 'tsExpire', 'uidAuthor', 'sDescription', 'sComment' ]; + + kcDbColumns = 7; + + def __init__(self): + ModelDataBase.__init__(self); + + # + # Initialize with defaults. + # See the database for explanations of each of these fields. + # + self.idTestGroup = None + self.tsEffective = None + self.tsExpire = None + self.uidAuthor = None + self.sName = None + self.sDescription = None + self.sComment = None + + def initFromDbRow(self, aoRow): + """ + Reinitialize from a SELECT * FROM TestGroups row. + Returns object of type TestGroupData. Raises exception if no row. + """ + if aoRow is None: + raise TMRowNotFound('Test group not found.') + + self.idTestGroup = aoRow[0] + self.tsEffective = aoRow[1] + self.tsExpire = aoRow[2] + self.uidAuthor = aoRow[3] + self.sName = aoRow[4] + self.sDescription = aoRow[5] + self.sComment = aoRow[6] + return self + + def initFromDbWithId(self, oDb, idTestGroup, tsNow = None, sPeriodBack = None): + """ + Initialize the object from the database. + """ + oDb.execute(self.formatSimpleNowAndPeriodQuery(oDb, + 'SELECT *\n' + 'FROM TestGroups\n' + 'WHERE idTestGroup = %s\n' + , ( idTestGroup,), tsNow, sPeriodBack)); + aoRow = oDb.fetchOne() + if aoRow is None: + raise TMRowNotFound('idTestGroup=%s not found (tsNow=%s sPeriodBack=%s)' % (idTestGroup, tsNow, sPeriodBack,)); + return self.initFromDbRow(aoRow); + + +class TestGroupDataEx(TestGroupData): + """ + Extended test group data. + """ + + ksParam_aoMembers = 'TestGroupDataEx_aoMembers'; + kasAltArrayNull = [ 'aoMembers', ]; + + ## Helper parameter containing the comma separated list with the IDs of + # potential members found in the parameters. + ksParam_aidTestCases = 'TestGroupDataEx_aidTestCases'; + + + def __init__(self): + TestGroupData.__init__(self); + self.aoMembers = []; # TestGroupMemberDataEx. + + def _initExtraMembersFromDb(self, oDb, tsNow = None, sPeriodBack = None): + """ + Worker shared by the initFromDb* methods. + Returns self. Raises exception if no row or database error. + """ + self.aoMembers = []; + _ = sPeriodBack; ## @todo sPeriodBack + + if tsNow is None: + oDb.execute('SELECT TestGroupMembers.*, TestCases.*\n' + 'FROM TestGroupMembers\n' + 'LEFT OUTER JOIN TestCases ON (\n' + ' TestGroupMembers.idTestCase = TestCases.idTestCase\n' + ' AND TestCases.tsExpire = \'infinity\'::TIMESTAMP)\n' + 'WHERE TestGroupMembers.idTestGroup = %s\n' + ' AND TestGroupMembers.tsExpire = \'infinity\'::TIMESTAMP\n' + 'ORDER BY TestCases.sName, TestCases.idTestCase\n' + , (self.idTestGroup,)); + else: + oDb.execute('SELECT TestGroupMembers.*, TestCases.*\n' + 'FROM TestGroupMembers\n' + 'LEFT OUTER JOIN TestCases ON (\n' + ' TestGroupMembers.idTestCase = TestCases.idTestCase\n' + ' AND TestCases.tsExpire > %s\n' + ' AND TestCases.tsEffective <= %s)\n' + 'WHERE TestGroupMembers.idTestGroup = %s\n' + ' AND TestGroupMembers.tsExpire > %s\n' + ' AND TestGroupMembers.tsEffective <= %s\n' + 'ORDER BY TestCases.sName, TestCases.idTestCase\n' + , (tsNow, tsNow, self.idTestGroup, tsNow, tsNow)); + + for aoRow in oDb.fetchAll(): + self.aoMembers.append(TestGroupMemberDataEx().initFromDbRowEx(aoRow, oDb, tsNow)); + return self; + + def initFromDbRowEx(self, aoRow, oDb, tsNow = None, sPeriodBack = None): + """ + Reinitialize from a SELECT * FROM TestGroups row. Will query the + necessary additional data from oDb using tsNow. + Returns self. Raises exception if no row or database error. + """ + TestGroupData.initFromDbRow(self, aoRow); + return self._initExtraMembersFromDb(oDb, tsNow, sPeriodBack); + + def initFromDbWithId(self, oDb, idTestGroup, tsNow = None, sPeriodBack = None): + """ + Initialize the object from the database. + """ + TestGroupData.initFromDbWithId(self, oDb, idTestGroup, tsNow, sPeriodBack); + return self._initExtraMembersFromDb(oDb, tsNow, sPeriodBack); + + + def getAttributeParamNullValues(self, sAttr): + if sAttr != 'aoMembers': + return TestGroupData.getAttributeParamNullValues(self, sAttr); + return ['', [], None]; + + def convertParamToAttribute(self, sAttr, sParam, oValue, oDisp, fStrict): + if sAttr != 'aoMembers': + return TestGroupData.convertParamToAttribute(self, sAttr, sParam, oValue, oDisp, fStrict); + + aoNewValue = []; + aidSelected = oDisp.getListOfIntParams(sParam, iMin = 1, iMax = 0x7ffffffe, aiDefaults = []) + sIds = oDisp.getStringParam(self.ksParam_aidTestCases, sDefault = ''); + for idTestCase in sIds.split(','): + try: idTestCase = int(idTestCase); + except: pass; + oDispWrapper = self.DispWrapper(oDisp, '%s[%s][%%s]' % (TestGroupDataEx.ksParam_aoMembers, idTestCase,)) + oMember = TestGroupMemberDataEx().initFromParams(oDispWrapper, fStrict = False); + if idTestCase in aidSelected: + aoNewValue.append(oMember); + return aoNewValue; + + def _validateAndConvertAttribute(self, sAttr, sParam, oValue, aoNilValues, fAllowNull, oDb): + if sAttr != 'aoMembers': + return TestGroupData._validateAndConvertAttribute(self, sAttr, sParam, oValue, aoNilValues, fAllowNull, oDb); + + asErrors = []; + aoNewMembers = []; + for oOldMember in oValue: + oNewMember = TestGroupMemberDataEx().initFromOther(oOldMember); + aoNewMembers.append(oNewMember); + + dErrors = oNewMember.validateAndConvert(oDb, ModelDataBase.ksValidateFor_Other); + if dErrors: + asErrors.append(str(dErrors)); + + if not asErrors: + for i, _ in enumerate(aoNewMembers): + idTestCase = aoNewMembers[i]; + for j in range(i + 1, len(aoNewMembers)): + if aoNewMembers[j].idTestCase == idTestCase: + asErrors.append('Duplicate testcase #%d!' % (idTestCase, )); + break; + + return (aoNewMembers, None if not asErrors else '<br>\n'.join(asErrors)); + + +class TestGroupLogic(ModelLogicBase): + """ + Test case management logic. + """ + + def __init__(self, oDb): + ModelLogicBase.__init__(self, oDb) + self.dCache = None; + + # + # Standard methods. + # + + def fetchForListing(self, iStart, cMaxRows, tsNow, aiSortColumns = None): + """ + Fetches test groups. + + Returns an array (list) of TestGroupDataEx items, empty list if none. + Raises exception on error. + """ + _ = aiSortColumns; + if tsNow is None: + self._oDb.execute('SELECT *\n' + 'FROM TestGroups\n' + 'WHERE tsExpire = \'infinity\'::TIMESTAMP\n' + 'ORDER BY sName ASC\n' + 'LIMIT %s OFFSET %s\n' + , (cMaxRows, iStart,)); + else: + self._oDb.execute('SELECT *\n' + 'FROM TestGroups\n' + 'WHERE tsExpire > %s\n' + ' AND tsEffective <= %s\n' + 'ORDER BY sName ASC\n' + 'LIMIT %s OFFSET %s\n' + , (tsNow, tsNow, cMaxRows, iStart,)); + + aoRet = []; + for aoRow in self._oDb.fetchAll(): + aoRet.append(TestGroupDataEx().initFromDbRowEx(aoRow, self._oDb, tsNow)); + return aoRet; + + def addEntry(self, oData, uidAuthor, fCommit = False): + """ + Adds a testgroup to the database. + """ + + # + # Validate inputs. + # + assert isinstance(oData, TestGroupDataEx); + dErrors = oData.validateAndConvert(self._oDb, oData.ksValidateFor_Add); + if dErrors: + raise TMInvalidData('addEntry invalid input: %s' % (dErrors,)); + self._assertUniq(oData, None); + + # + # Do the job. + # + self._oDb.execute('INSERT INTO TestGroups (uidAuthor, sName, sDescription, sComment)\n' + 'VALUES (%s, %s, %s, %s)\n' + 'RETURNING idTestGroup\n' + , ( uidAuthor, + oData.sName, + oData.sDescription, + oData.sComment )); + idTestGroup = self._oDb.fetchOne()[0]; + oData.idTestGroup = idTestGroup; + + for oMember in oData.aoMembers: + oMember.idTestGroup = idTestGroup; + self._insertTestGroupMember(uidAuthor, oMember) + + self._oDb.maybeCommit(fCommit); + return True; + + def editEntry(self, oData, uidAuthor, fCommit = False): + """ + Modifies a test group. + """ + + # + # Validate inputs and read in the old(/current) data. + # + assert isinstance(oData, TestGroupDataEx); + dErrors = oData.validateAndConvert(self._oDb, oData.ksValidateFor_Edit); + if dErrors: + raise TMInvalidData('editEntry invalid input: %s' % (dErrors,)); + self._assertUniq(oData, oData.idTestGroup); + + oOldData = TestGroupDataEx().initFromDbWithId(self._oDb, oData.idTestGroup); + + # + # Update the data that needs updating. + # + + if not oData.isEqualEx(oOldData, [ 'aoMembers', 'tsEffective', 'tsExpire', 'uidAuthor', ]): + self._historizeTestGroup(oData.idTestGroup); + self._oDb.execute('INSERT INTO TestGroups\n' + ' (uidAuthor, idTestGroup, sName, sDescription, sComment)\n' + 'VALUES (%s, %s, %s, %s, %s)\n' + , ( uidAuthor, + oData.idTestGroup, + oData.sName, + oData.sDescription, + oData.sComment )); + + # Create a lookup dictionary for old entries. + dOld = {}; + for oOld in oOldData.aoMembers: + dOld[oOld.idTestCase] = oOld; + assert len(dOld) == len(oOldData.aoMembers); + + # Add new members, updated existing ones. + dNew = {}; + for oNewMember in oData.aoMembers: + oNewMember.idTestGroup = oData.idTestGroup; + if oNewMember.idTestCase in dNew: + raise TMRowAlreadyExists('Duplicate test group member: idTestCase=%d (%s / %s)' + % (oNewMember.idTestCase, oNewMember, dNew[oNewMember.idTestCase],)); + dNew[oNewMember.idTestCase] = oNewMember; + + oOldMember = dOld.get(oNewMember.idTestCase, None); + if oOldMember is not None: + if oNewMember.isEqualEx(oOldMember, [ 'uidAuthor', 'tsEffective', 'tsExpire' ]): + continue; # Skip, nothing changed. + self._historizeTestGroupMember(oData.idTestGroup, oNewMember.idTestCase); + self._insertTestGroupMember(uidAuthor, oNewMember); + + # Expire members that have been removed. + sQuery = self._oDb.formatBindArgs('UPDATE TestGroupMembers\n' + 'SET tsExpire = CURRENT_TIMESTAMP\n' + 'WHERE idTestGroup = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , ( oData.idTestGroup, )); + if dNew: + sQuery += ' AND idTestCase NOT IN (%s)' % (', '.join([str(iKey) for iKey in dNew]),); + self._oDb.execute(sQuery); + + self._oDb.maybeCommit(fCommit); + return True; + + def removeEntry(self, uidAuthor, idTestGroup, fCascade = False, fCommit = False): + """ + Deletes a test group. + """ + _ = uidAuthor; ## @todo record uidAuthor. + + # + # Cascade. + # + if fCascade is not True: + self._oDb.execute('SELECT SchedGroups.idSchedGroup, SchedGroups.sName\n' + 'FROM SchedGroupMembers, SchedGroups\n' + 'WHERE SchedGroupMembers.idTestGroup = %s\n' + ' AND SchedGroupMembers.tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND SchedGroups.idSchedGroup = SchedGroupMembers.idSchedGroup\n' + ' AND SchedGroups.tsExpire = \'infinity\'::TIMESTAMP\n' + , ( idTestGroup, )); + aoGroups = self._oDb.fetchAll(); + if aoGroups: + asGroups = ['%s (#%d)' % (sName, idSchedGroup) for idSchedGroup, sName in aoGroups]; + raise TMRowInUse('Test group #%d is member of one or more scheduling groups: %s' + % (idTestGroup, ', '.join(asGroups),)); + else: + self._oDb.execute('UPDATE SchedGroupMembers\n' + 'SET tsExpire = CURRENT_TIMESTAMP\n' + 'WHERE idTestGroup = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , ( idTestGroup, )); + + # + # Remove the group. + # + self._oDb.execute('UPDATE TestGroupMembers\n' + 'SET tsExpire = CURRENT_TIMESTAMP\n' + 'WHERE idTestGroup = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , (idTestGroup,)) + self._oDb.execute('UPDATE TestGroups\n' + 'SET tsExpire = CURRENT_TIMESTAMP\n' + 'WHERE idTestGroup = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , (idTestGroup,)) + + self._oDb.maybeCommit(fCommit) + return True; + + def cachedLookup(self, idTestGroup): + """ + Looks up the most recent TestGroupDataEx object for idTestGroup + via an object cache. + + Returns a shared TestGroupDataEx object. None if not found. + Raises exception on DB error. + """ + if self.dCache is None: + self.dCache = self._oDb.getCache('TestGroupDataEx'); + oEntry = self.dCache.get(idTestGroup, None); + if oEntry is None: + fNeedTsNow = False; + self._oDb.execute('SELECT *\n' + 'FROM TestGroups\n' + 'WHERE idTestGroup = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , (idTestGroup, )); + if self._oDb.getRowCount() == 0: + # Maybe it was deleted, try get the last entry. + self._oDb.execute('SELECT *\n' + 'FROM TestGroups\n' + 'WHERE idTestGroup = %s\n' + 'ORDER BY tsExpire DESC\n' + 'LIMIT 1\n' + , (idTestGroup, )); + fNeedTsNow = True; + elif self._oDb.getRowCount() > 1: + raise self._oDb.integrityException('%s infinity rows for %s' % (self._oDb.getRowCount(), idTestGroup)); + + if self._oDb.getRowCount() == 1: + aaoRow = self._oDb.fetchOne(); + oEntry = TestGroupDataEx(); + tsNow = oEntry.initFromDbRow(aaoRow).tsEffective if fNeedTsNow else None; + oEntry.initFromDbRowEx(aaoRow, self._oDb, tsNow); + self.dCache[idTestGroup] = oEntry; + return oEntry; + + + # + # Other methods. + # + + def fetchOrderedByName(self, tsNow = None): + """ + Return list of objects of type TestGroupData ordered by name. + May raise exception on database error. + """ + if tsNow is None: + self._oDb.execute('SELECT *\n' + 'FROM TestGroups\n' + 'WHERE tsExpire = \'infinity\'::TIMESTAMP\n' + 'ORDER BY sName ASC\n'); + else: + self._oDb.execute('SELECT *\n' + 'FROM TestGroups\n' + 'WHERE tsExpire > %s\n' + ' AND tsEffective <= %s\n' + 'ORDER BY sName ASC\n' + , (tsNow, tsNow,)); + aoRet = [] + for _ in range(self._oDb.getRowCount()): + aoRet.append(TestGroupData().initFromDbRow(self._oDb.fetchOne())); + return aoRet; + + def getMembers(self, idTestGroup): + """ + Fetches all test case records from DB which are + belong to current Test Group. + Returns list of objects of type TestGroupMemberData2 (!). + """ + self._oDb.execute('SELECT TestCases.*,\n' + ' TestGroupMembers.idTestGroup,\n' + ' TestGroupMembers.aidTestCaseArgs\n' + 'FROM TestCases, TestGroupMembers\n' + 'WHERE TestCases.tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND TestGroupMembers.tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND TestGroupMembers.idTestCase = TestCases.idTestCase\n' + ' AND TestGroupMembers.idTestGroup = %s\n' + 'ORDER BY TestCases.idTestCase ASC;', + (idTestGroup,)) + + aaoRows = self._oDb.fetchAll() + aoRet = [] + for aoRow in aaoRows: + aoRet.append(TestGroupMemberData2().initFromDbRowEx(aoRow)) + + return aoRet + + def getAll(self, tsNow=None): + """Return list of objects of type TestGroupData""" + + if tsNow is None: + self._oDb.execute('SELECT *\n' + 'FROM TestGroups\n' + 'WHERE tsExpire = \'infinity\'::TIMESTAMP\n' + 'ORDER BY idTestGroup ASC;') + else: + self._oDb.execute('SELECT *\n' + 'FROM TestGroups\n' + 'WHERE tsExpire > %s\n' + ' AND tsEffective <= %s\n' + 'ORDER BY idTestGroup ASC;', + (tsNow, tsNow)) + + aaoRows = self._oDb.fetchAll() + aoRet = [] + for aoRow in aaoRows: + aoRet.append(TestGroupData().initFromDbRow(aoRow)) + + return aoRet + + def getById(self, idTestGroup, tsNow=None): + """Get Test Group data by its ID""" + + if tsNow is None: + self._oDb.execute('SELECT *\n' + 'FROM TestGroups\n' + 'WHERE tsExpire = \'infinity\'::timestamp\n' + ' AND idTestGroup = %s\n' + 'ORDER BY idTestGroup ASC;' + , (idTestGroup,)) + else: + self._oDb.execute('SELECT *\n' + 'FROM TestGroups\n' + 'WHERE tsExpire > %s\n' + ' AND tsEffective <= %s\n' + ' AND idTestGroup = %s\n' + 'ORDER BY idTestGroup ASC;' + , (tsNow, tsNow, idTestGroup)) + + aRows = self._oDb.fetchAll() + if len(aRows) not in (0, 1): + raise TMTooManyRows('Found more than one test groups with the same credentials. Database structure is corrupted.') + try: + return TestGroupData().initFromDbRow(aRows[0]) + except IndexError: + return None + + # + # Helpers. + # + + def _assertUniq(self, oData, idTestGroupIgnore): + """ Checks that the test group name is unique, raises exception if it isn't. """ + self._oDb.execute('SELECT idTestGroup\n' + 'FROM TestGroups\n' + 'WHERE sName = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + + ('' if idTestGroupIgnore is None else ' AND idTestGroup <> %d\n' % (idTestGroupIgnore,)) + , ( oData.sName, )) + if self._oDb.getRowCount() > 0: + raise TMRowAlreadyExists('A Test group with name "%s" already exist.' % (oData.sName,)); + return True; + + def _historizeTestGroup(self, idTestGroup): + """ Historize Test Group record. """ + self._oDb.execute('UPDATE TestGroups\n' + 'SET tsExpire = CURRENT_TIMESTAMP\n' + 'WHERE idTestGroup = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , ( idTestGroup, )); + return True; + + def _historizeTestGroupMember(self, idTestGroup, idTestCase): + """ Historize Test Group Member record. """ + self._oDb.execute('UPDATE TestGroupMembers\n' + 'SET tsExpire = CURRENT_TIMESTAMP\n' + 'WHERE idTestGroup = %s\n' + ' AND idTestCase = %s\n' + ' AND tsExpire = \'infinity\'::timestamp\n' + , (idTestGroup, idTestCase,)); + return True; + + def _insertTestGroupMember(self, uidAuthor, oMember): + """ Inserts a test group member. """ + self._oDb.execute('INSERT INTO TestGroupMembers\n' + ' (uidAuthor, idTestGroup, idTestCase, iSchedPriority, aidTestCaseArgs)\n' + 'VALUES (%s, %s, %s, %s, %s)\n' + , ( uidAuthor, + oMember.idTestGroup, + oMember.idTestCase, + oMember.iSchedPriority, + oMember.aidTestCaseArgs, )); + return True; + + + +# +# Unit testing. +# + +# pylint: disable=missing-docstring +class TestGroupMemberDataTestCase(ModelDataBaseTestCase): + def setUp(self): + self.aoSamples = [TestGroupMemberData(),]; + +class TestGroupDataTestCase(ModelDataBaseTestCase): + def setUp(self): + self.aoSamples = [TestGroupData(),]; + +if __name__ == '__main__': + unittest.main(); + # not reached. + diff --git a/src/VBox/ValidationKit/testmanager/core/testresultfailures.py b/src/VBox/ValidationKit/testmanager/core/testresultfailures.py new file mode 100755 index 00000000..b94ff7d5 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/testresultfailures.py @@ -0,0 +1,529 @@ +# -*- coding: utf-8 -*- +# $Id: testresultfailures.py $ +# pylint: disable=too-many-lines + +## @todo Rename this file to testresult.py! + +""" +Test Manager - Test result failures. +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + +# Standard python imports. +import sys; +import unittest; + +# Validation Kit imports. +from testmanager.core.base import ModelDataBase, ModelLogicBase, ModelDataBaseTestCase, TMInvalidData, TMRowNotFound, \ + TMRowAlreadyExists, ChangeLogEntry, AttributeChangeEntry; +from testmanager.core.failurereason import FailureReasonData; +from testmanager.core.useraccount import UserAccountLogic; + +# Python 3 hacks: +if sys.version_info[0] >= 3: + xrange = range; # pylint: disable=redefined-builtin,invalid-name + + +class TestResultFailureData(ModelDataBase): + """ + Test result failure reason data. + """ + + ksIdAttr = 'idTestResult'; + kfIdAttrIsForForeign = True; # Modifies the 'add' validation. + + ksParam_idTestResult = 'TestResultFailure_idTestResult'; + ksParam_tsEffective = 'TestResultFailure_tsEffective'; + ksParam_tsExpire = 'TestResultFailure_tsExpire'; + ksParam_uidAuthor = 'TestResultFailure_uidAuthor'; + ksParam_idTestSet = 'TestResultFailure_idTestSet'; + ksParam_idFailureReason = 'TestResultFailure_idFailureReason'; + ksParam_sComment = 'TestResultFailure_sComment'; + + kasAllowNullAttributes = ['tsEffective', 'tsExpire', 'uidAuthor', 'sComment', 'idTestSet' ]; + + kcDbColumns = 7; + + def __init__(self): + ModelDataBase.__init__(self) + self.idTestResult = None; + self.tsEffective = None; + self.tsExpire = None; + self.uidAuthor = None; + self.idTestSet = None; + self.idFailureReason = None; + self.sComment = None; + + def initFromDbRow(self, aoRow): + """ + Reinitialize from a SELECT * FROM TestResultFailures. + Return self. Raises exception if no row. + """ + if aoRow is None: + raise TMRowNotFound('Test result file record not found.') + + self.idTestResult = aoRow[0]; + self.tsEffective = aoRow[1]; + self.tsExpire = aoRow[2]; + self.uidAuthor = aoRow[3]; + self.idTestSet = aoRow[4]; + self.idFailureReason = aoRow[5]; + self.sComment = aoRow[6]; + return self; + + def initFromDbWithId(self, oDb, idTestResult, tsNow = None, sPeriodBack = None): + """ + Initialize the object from the database. + """ + oDb.execute(self.formatSimpleNowAndPeriodQuery(oDb, + 'SELECT *\n' + 'FROM TestResultFailures\n' + 'WHERE idTestResult = %s\n' + , ( idTestResult,), tsNow, sPeriodBack)); + aoRow = oDb.fetchOne() + if aoRow is None: + raise TMRowNotFound('idTestResult=%s not found (tsNow=%s, sPeriodBack=%s)' % (idTestResult, tsNow, sPeriodBack)); + assert len(aoRow) == self.kcDbColumns; + return self.initFromDbRow(aoRow); + + def initFromValues(self, idTestResult, idFailureReason, uidAuthor, + tsExpire = None, tsEffective = None, idTestSet = None, sComment = None): + """ + Initialize from values. + """ + self.idTestResult = idTestResult; + self.tsEffective = tsEffective; + self.tsExpire = tsExpire; + self.uidAuthor = uidAuthor; + self.idTestSet = idTestSet; + self.idFailureReason = idFailureReason; + self.sComment = sComment; + return self; + + + +class TestResultFailureDataEx(TestResultFailureData): + """ + Extends TestResultFailureData by resolving reasons and user. + """ + + def __init__(self): + TestResultFailureData.__init__(self); + self.oFailureReason = None; + self.oAuthor = None; + + def initFromDbRowEx(self, aoRow, oFailureReasonLogic, oUserAccountLogic): + """ + Reinitialize from a SELECT * FROM TestResultFailures. + Return self. Raises exception if no row. + """ + self.initFromDbRow(aoRow); + self.oFailureReason = oFailureReasonLogic.cachedLookup(self.idFailureReason); + self.oAuthor = oUserAccountLogic.cachedLookup(self.uidAuthor); + return self; + + +class TestResultListingData(ModelDataBase): # pylint: disable=too-many-instance-attributes + """ + Test case result data representation for table listing + """ + + def __init__(self): + """Initialize""" + ModelDataBase.__init__(self) + + self.idTestSet = None + + self.idBuildCategory = None; + self.sProduct = None + self.sRepository = None; + self.sBranch = None + self.sType = None + self.idBuild = None; + self.sVersion = None; + self.iRevision = None + + self.sOs = None; + self.sOsVersion = None; + self.sArch = None; + self.sCpuVendor = None; + self.sCpuName = None; + self.cCpus = None; + self.fCpuHwVirt = None; + self.fCpuNestedPaging = None; + self.fCpu64BitGuest = None; + self.idTestBox = None + self.sTestBoxName = None + + self.tsCreated = None + self.tsElapsed = None + self.enmStatus = None + self.cErrors = None; + + self.idTestCase = None + self.sTestCaseName = None + self.sBaseCmd = None + self.sArgs = None + self.sSubName = None; + + self.idBuildTestSuite = None; + self.iRevisionTestSuite = None; + + self.oFailureReason = None; + self.oFailureReasonAssigner = None; + self.tsFailureReasonAssigned = None; + self.sFailureReasonComment = None; + + def initFromDbRowEx(self, aoRow, oFailureReasonLogic, oUserAccountLogic): + """ + Reinitialize from a database query. + Return self. Raises exception if no row. + """ + if aoRow is None: + raise TMRowNotFound('Test result record not found.') + + self.idTestSet = aoRow[0]; + + self.idBuildCategory = aoRow[1]; + self.sProduct = aoRow[2]; + self.sRepository = aoRow[3]; + self.sBranch = aoRow[4]; + self.sType = aoRow[5]; + self.idBuild = aoRow[6]; + self.sVersion = aoRow[7]; + self.iRevision = aoRow[8]; + + self.sOs = aoRow[9]; + self.sOsVersion = aoRow[10]; + self.sArch = aoRow[11]; + self.sCpuVendor = aoRow[12]; + self.sCpuName = aoRow[13]; + self.cCpus = aoRow[14]; + self.fCpuHwVirt = aoRow[15]; + self.fCpuNestedPaging = aoRow[16]; + self.fCpu64BitGuest = aoRow[17]; + self.idTestBox = aoRow[18]; + self.sTestBoxName = aoRow[19]; + + self.tsCreated = aoRow[20]; + self.tsElapsed = aoRow[21]; + self.enmStatus = aoRow[22]; + self.cErrors = aoRow[23]; + + self.idTestCase = aoRow[24]; + self.sTestCaseName = aoRow[25]; + self.sBaseCmd = aoRow[26]; + self.sArgs = aoRow[27]; + self.sSubName = aoRow[28]; + + self.idBuildTestSuite = aoRow[29]; + self.iRevisionTestSuite = aoRow[30]; + + self.oFailureReason = None; + if aoRow[31] is not None: + self.oFailureReason = oFailureReasonLogic.cachedLookup(aoRow[31]); + self.oFailureReasonAssigner = None; + if aoRow[32] is not None: + self.oFailureReasonAssigner = oUserAccountLogic.cachedLookup(aoRow[32]); + self.tsFailureReasonAssigned = aoRow[33]; + self.sFailureReasonComment = aoRow[34]; + + return self + + + +class TestResultFailureLogic(ModelLogicBase): # pylint: disable=too-few-public-methods + """ + Test result failure reason logic. + """ + + def __init__(self, oDb): + ModelLogicBase.__init__(self, oDb) + + def fetchForChangeLog(self, idTestResult, iStart, cMaxRows, tsNow): # pylint: disable=too-many-locals + """ + Fetches change log entries for a failure reason. + + Returns an array of ChangeLogEntry instance and an indicator whether + there are more entries. + Raises exception on error. + """ + + if tsNow is None: + tsNow = self._oDb.getCurrentTimestamp(); + + # 1. Get a list of the changes from both TestResultFailures and assoicated + # FailureReasons. The latter is useful since the failure reason + # description may evolve along side the invidiual failure analysis. + self._oDb.execute('( SELECT trf.tsEffective AS tsEffectiveChangeLog,\n' + ' trf.uidAuthor AS uidAuthorChangeLog,\n' + ' trf.*,\n' + ' fr.*\n' + ' FROM TestResultFailures trf,\n' + ' FailureReasons fr\n' + ' WHERE trf.idTestResult = %s\n' + ' AND trf.tsEffective <= %s\n' + ' AND trf.idFailureReason = fr.idFailureReason\n' + ' AND fr.tsEffective <= trf.tsEffective\n' + ' AND fr.tsExpire > trf.tsEffective\n' + ')\n' + 'UNION\n' + '( SELECT fr.tsEffective AS tsEffectiveChangeLog,\n' + ' fr.uidAuthor AS uidAuthorChangeLog,\n' + ' trf.*,\n' + ' fr.*\n' + ' FROM TestResultFailures trf,\n' + ' FailureReasons fr\n' + ' WHERE trf.idTestResult = %s\n' + ' AND trf.tsEffective <= %s\n' + ' AND trf.idFailureReason = fr.idFailureReason\n' + ' AND fr.tsEffective > trf.tsEffective\n' + ' AND fr.tsEffective < trf.tsExpire\n' + ')\n' + 'ORDER BY tsEffectiveChangeLog DESC\n' + 'LIMIT %s OFFSET %s\n' + , ( idTestResult, tsNow, idTestResult, tsNow, cMaxRows + 1, iStart, )); + + aaoRows = []; + for aoChange in self._oDb.fetchAll(): + oTrf = TestResultFailureDataEx().initFromDbRow(aoChange[2:]); + oFr = FailureReasonData().initFromDbRow(aoChange[(2+TestResultFailureData.kcDbColumns):]); + oTrf.oFailureReason = oFr; + aaoRows.append([aoChange[0], aoChange[1], oTrf, oFr]); + + # 2. Calculate the changes. + oFailureCategoryLogic = None; + aoEntries = []; + for i in xrange(0, len(aaoRows) - 1): + aoNew = aaoRows[i]; + aoOld = aaoRows[i + 1]; + + aoChanges = []; + oNew = aoNew[2]; + oOld = aoOld[2]; + for sAttr in oNew.getDataAttributes(): + if sAttr not in [ 'tsEffective', 'tsExpire', 'uidAuthor', 'oFailureReason', 'oAuthor' ]: + oOldAttr = getattr(oOld, sAttr); + oNewAttr = getattr(oNew, sAttr); + if oOldAttr != oNewAttr: + if sAttr == 'idFailureReason': + oNewAttr = '%s (%s)' % (oNewAttr, oNew.oFailureReason.sShort, ); + oOldAttr = '%s (%s)' % (oOldAttr, oOld.oFailureReason.sShort, ); + aoChanges.append(AttributeChangeEntry(sAttr, oNewAttr, oOldAttr, str(oNewAttr), str(oOldAttr))); + if oOld.idFailureReason == oNew.idFailureReason: + oNew = aoNew[3]; + oOld = aoOld[3]; + for sAttr in oNew.getDataAttributes(): + if sAttr not in [ 'tsEffective', 'tsExpire', 'uidAuthor', ]: + oOldAttr = getattr(oOld, sAttr); + oNewAttr = getattr(oNew, sAttr); + if oOldAttr != oNewAttr: + if sAttr == 'idFailureCategory': + if oFailureCategoryLogic is None: + from testmanager.core.failurecategory import FailureCategoryLogic; + oFailureCategoryLogic = FailureCategoryLogic(self._oDb); + oCat = oFailureCategoryLogic.cachedLookup(oNewAttr); + if oCat is not None: + oNewAttr = '%s (%s)' % (oNewAttr, oCat.sShort, ); + oCat = oFailureCategoryLogic.cachedLookup(oOldAttr); + if oCat is not None: + oOldAttr = '%s (%s)' % (oOldAttr, oCat.sShort, ); + aoChanges.append(AttributeChangeEntry(sAttr, oNewAttr, oOldAttr, str(oNewAttr), str(oOldAttr))); + + + tsExpire = aaoRows[i - 1][0] if i > 0 else aoNew[2].tsExpire; + aoEntries.append(ChangeLogEntry(aoNew[1], None, aoNew[0], tsExpire, aoNew[2], aoOld[2], aoChanges)); + + # If we're at the end of the log, add the initial entry. + if len(aaoRows) <= cMaxRows and aaoRows: + aoNew = aaoRows[-1]; + tsExpire = aaoRows[-1 - 1][0] if len(aaoRows) > 1 else aoNew[2].tsExpire; + aoEntries.append(ChangeLogEntry(aoNew[1], None, aoNew[0], tsExpire, aoNew[2], None, [])); + + return (UserAccountLogic(self._oDb).resolveChangeLogAuthors(aoEntries), len(aaoRows) > cMaxRows); + + + def getById(self, idTestResult): + """Get Test result failure reason data by idTestResult""" + + self._oDb.execute('SELECT *\n' + 'FROM TestResultFailures\n' + 'WHERE tsExpire = \'infinity\'::timestamp\n' + ' AND idTestResult = %s;', (idTestResult,)) + aRows = self._oDb.fetchAll() + if len(aRows) not in (0, 1): + raise self._oDb.integrityException( + 'Found more than one failure reasons with the same credentials. Database structure is corrupted.') + try: + return TestResultFailureData().initFromDbRow(aRows[0]) + except IndexError: + return None + + def addEntry(self, oData, uidAuthor, fCommit = False): + """ + Add a test result failure reason record. + """ + + # + # Validate inputs and read in the old(/current) data. + # + assert isinstance(oData, TestResultFailureData); + dErrors = oData.validateAndConvert(self._oDb, oData.ksValidateFor_AddForeignId); + if dErrors: + raise TMInvalidData('editEntry invalid input: %s' % (dErrors,)); + + # Check if it exist first (we're adding, not editing, collisions not allowed). + oOldData = self.getById(oData.idTestResult); + if oOldData is not None: + raise TMRowAlreadyExists('TestResult %d already have a failure reason associated with it:' + '%s\n' + 'Perhaps someone else beat you to it? Or did you try resubmit?' + % (oData.idTestResult, oOldData)); + oData = self._resolveSetTestIdIfMissing(oData); + + # + # Add record. + # + self._readdEntry(uidAuthor, oData); + self._oDb.maybeCommit(fCommit); + return True; + + def editEntry(self, oData, uidAuthor, fCommit = False): + """ + Modifies a test result failure reason. + """ + + # + # Validate inputs and read in the old(/current) data. + # + assert isinstance(oData, TestResultFailureData); + dErrors = oData.validateAndConvert(self._oDb, oData.ksValidateFor_Edit); + if dErrors: + raise TMInvalidData('editEntry invalid input: %s' % (dErrors,)); + + oOldData = self.getById(oData.idTestResult) + oData.idTestSet = oOldData.idTestSet; + + # + # Update the data that needs updating. + # + if not oData.isEqualEx(oOldData, [ 'tsEffective', 'tsExpire', 'uidAuthor', ]): + self._historizeEntry(oData.idTestResult); + self._readdEntry(uidAuthor, oData); + self._oDb.maybeCommit(fCommit); + return True; + + + def removeEntry(self, uidAuthor, idTestResult, fCascade = False, fCommit = False): + """ + Deletes a test result failure reason. + """ + _ = fCascade; # Not applicable. + + oData = self.getById(idTestResult) + (tsCur, tsCurMinusOne) = self._oDb.getCurrentTimestamps(); + if oData.tsEffective not in (tsCur, tsCurMinusOne): + self._historizeEntry(idTestResult, tsCurMinusOne); + self._readdEntry(uidAuthor, oData, tsCurMinusOne); + self._historizeEntry(idTestResult); + self._oDb.execute('UPDATE TestResultFailures\n' + 'SET tsExpire = CURRENT_TIMESTAMP\n' + 'WHERE idTestResult = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , (idTestResult,)); + self._oDb.maybeCommit(fCommit); + return True; + + # + # Helpers. + # + + def _readdEntry(self, uidAuthor, oData, tsEffective = None): + """ + Re-adds the TestResultFailure entry. Used by addEntry, editEntry and removeEntry. + """ + if tsEffective is None: + tsEffective = self._oDb.getCurrentTimestamp(); + self._oDb.execute('INSERT INTO TestResultFailures (\n' + ' uidAuthor,\n' + ' tsEffective,\n' + ' idTestResult,\n' + ' idTestSet,\n' + ' idFailureReason,\n' + ' sComment)\n' + 'VALUES (%s, %s, %s, %s, %s, %s)\n' + , ( uidAuthor, + tsEffective, + oData.idTestResult, + oData.idTestSet, + oData.idFailureReason, + oData.sComment,) ); + return True; + + + def _historizeEntry(self, idTestResult, tsExpire = None): + """ Historizes the current entry. """ + if tsExpire is None: + tsExpire = self._oDb.getCurrentTimestamp(); + self._oDb.execute('UPDATE TestResultFailures\n' + 'SET tsExpire = %s\n' + 'WHERE idTestResult = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , (tsExpire, idTestResult,)); + return True; + + + def _resolveSetTestIdIfMissing(self, oData): + """ Resolve any missing idTestSet reference (it's a duplicate for speed efficiency). """ + if oData.idTestSet is None and oData.idTestResult is not None: + self._oDb.execute('SELECT idTestSet FROM TestResults WHERE idTestResult = %s', (oData.idTestResult,)); + oData.idTestSet = self._oDb.fetchOne()[0]; + return oData; + + + +# +# Unit testing. +# + +# pylint: disable=missing-docstring +class TestResultFailureDataTestCase(ModelDataBaseTestCase): + def setUp(self): + self.aoSamples = [TestResultFailureData(),]; + +if __name__ == '__main__': + unittest.main(); + # not reached. + diff --git a/src/VBox/ValidationKit/testmanager/core/testresults.py b/src/VBox/ValidationKit/testmanager/core/testresults.py new file mode 100755 index 00000000..58f056d2 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/testresults.py @@ -0,0 +1,2926 @@ +# -*- coding: utf-8 -*- +# $Id: testresults.py $ +# pylint: disable=too-many-lines + +## @todo Rename this file to testresult.py! + +""" +Test Manager - Fetch test results. +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Standard python imports. +import sys; +import unittest; + +# Validation Kit imports. +from common import constants; +from testmanager import config; +from testmanager.core.base import ModelDataBase, ModelLogicBase, ModelDataBaseTestCase, ModelFilterBase, \ + FilterCriterion, FilterCriterionValueAndDescription, \ + TMExceptionBase, TMTooManyRows, TMRowNotFound; +from testmanager.core.testgroup import TestGroupData; +from testmanager.core.build import BuildDataEx, BuildCategoryData; +from testmanager.core.failurereason import FailureReasonLogic; +from testmanager.core.testbox import TestBoxData, TestBoxLogic; +from testmanager.core.testcase import TestCaseData; +from testmanager.core.schedgroup import SchedGroupData, SchedGroupLogic; +from testmanager.core.systemlog import SystemLogData, SystemLogLogic; +from testmanager.core.testresultfailures import TestResultFailureDataEx; +from testmanager.core.useraccount import UserAccountLogic; + +# Python 3 hacks: +if sys.version_info[0] >= 3: + long = int; # pylint: disable=redefined-builtin,invalid-name + + +class TestResultData(ModelDataBase): + """ + Test case execution result data + """ + + ## @name TestStatus_T + # @{ + ksTestStatus_Running = 'running'; + ksTestStatus_Success = 'success'; + ksTestStatus_Skipped = 'skipped'; + ksTestStatus_BadTestBox = 'bad-testbox'; + ksTestStatus_Aborted = 'aborted'; + ksTestStatus_Failure = 'failure'; + ksTestStatus_TimedOut = 'timed-out'; + ksTestStatus_Rebooted = 'rebooted'; + ## @} + + ## List of relatively harmless (to testgroup/case) statuses. + kasHarmlessTestStatuses = [ ksTestStatus_Skipped, ksTestStatus_BadTestBox, ksTestStatus_Aborted, ]; + ## List of bad statuses. + kasBadTestStatuses = [ ksTestStatus_Failure, ksTestStatus_TimedOut, ksTestStatus_Rebooted, ]; + + + ksIdAttr = 'idTestResult'; + + ksParam_idTestResult = 'TestResultData_idTestResult'; + ksParam_idTestResultParent = 'TestResultData_idTestResultParent'; + ksParam_idTestSet = 'TestResultData_idTestSet'; + ksParam_tsCreated = 'TestResultData_tsCreated'; + ksParam_tsElapsed = 'TestResultData_tsElapsed'; + ksParam_idStrName = 'TestResultData_idStrName'; + ksParam_cErrors = 'TestResultData_cErrors'; + ksParam_enmStatus = 'TestResultData_enmStatus'; + ksParam_iNestingDepth = 'TestResultData_iNestingDepth'; + kasValidValues_enmStatus = [ + ksTestStatus_Running, + ksTestStatus_Success, + ksTestStatus_Skipped, + ksTestStatus_BadTestBox, + ksTestStatus_Aborted, + ksTestStatus_Failure, + ksTestStatus_TimedOut, + ksTestStatus_Rebooted + ]; + + + def __init__(self): + ModelDataBase.__init__(self) + self.idTestResult = None + self.idTestResultParent = None + self.idTestSet = None + self.tsCreated = None + self.tsElapsed = None + self.idStrName = None + self.cErrors = 0; + self.enmStatus = None + self.iNestingDepth = None + + def initFromDbRow(self, aoRow): + """ + Reinitialize from a SELECT * FROM TestResults. + Return self. Raises exception if no row. + """ + if aoRow is None: + raise TMRowNotFound('Test result record not found.') + + self.idTestResult = aoRow[0] + self.idTestResultParent = aoRow[1] + self.idTestSet = aoRow[2] + self.tsCreated = aoRow[3] + self.tsElapsed = aoRow[4] + self.idStrName = aoRow[5] + self.cErrors = aoRow[6] + self.enmStatus = aoRow[7] + self.iNestingDepth = aoRow[8] + return self; + + def initFromDbWithId(self, oDb, idTestResult, tsNow = None, sPeriodBack = None): + """ + Initialize from the database, given the ID of a row. + """ + _ = tsNow; + _ = sPeriodBack; + oDb.execute('SELECT *\n' + 'FROM TestResults\n' + 'WHERE idTestResult = %s\n' + , ( idTestResult,)); + aoRow = oDb.fetchOne() + if aoRow is None: + raise TMRowNotFound('idTestResult=%s not found' % (idTestResult,)); + return self.initFromDbRow(aoRow); + + def isFailure(self): + """ Check if it's a real failure. """ + return self.enmStatus in self.kasBadTestStatuses; + + +class TestResultDataEx(TestResultData): + """ + Extended test result data class. + + This is intended for use as a node in a result tree. This is not intended + for serialization to parameters or vice versa. Use TestResultLogic to + construct the tree. + """ + + def __init__(self): + TestResultData.__init__(self) + self.sName = None; # idStrName resolved. + self.oParent = None; # idTestResultParent within the tree. + + self.aoChildren = []; # TestResultDataEx; + self.aoValues = []; # TestResultValueDataEx; + self.aoMsgs = []; # TestResultMsgDataEx; + self.aoFiles = []; # TestResultFileDataEx; + self.oReason = None; # TestResultReasonDataEx; + + def initFromDbRow(self, aoRow): + """ + Initialize from a query like this: + SELECT TestResults.*, TestResultStrTab.sValue + FROM TestResults, TestResultStrTab + WHERE TestResultStrTab.idStr = TestResults.idStrName + + Note! The caller is expected to fetch children, values, failure + details, and files. + """ + self.sName = None; + self.oParent = None; + self.aoChildren = []; + self.aoValues = []; + self.aoMsgs = []; + self.aoFiles = []; + self.oReason = None; + + TestResultData.initFromDbRow(self, aoRow); + + self.sName = aoRow[9]; + return self; + + def deepCountErrorContributers(self): + """ + Counts how many test result instances actually contributed to cErrors. + """ + + # Check each child (if any). + cChanges = 0; + cChildErrors = 0; + for oChild in self.aoChildren: + if oChild.cErrors > 0: + cChildErrors += oChild.cErrors; + cChanges += oChild.deepCountErrorContributers(); + + # Did we contribute as well? + if self.cErrors > cChildErrors: + cChanges += 1; + return cChanges; + + def getListOfFailures(self): + """ + Get a list of test results instances actually contributing to cErrors. + + Returns a list of TestResultDataEx instances from this tree. (shared!) + """ + # Check each child (if any). + aoRet = []; + cChildErrors = 0; + for oChild in self.aoChildren: + if oChild.cErrors > 0: + cChildErrors += oChild.cErrors; + aoRet.extend(oChild.getListOfFailures()); + + # Did we contribute as well? + if self.cErrors > cChildErrors: + aoRet.append(self); + + return aoRet; + + def getListOfLogFilesByKind(self, asKinds): + """ + Get a list of test results instances actually contributing to cErrors. + + Returns a list of TestResultFileDataEx instances from this tree. (shared!) + """ + aoRet = []; + + # Check the children first. + for oChild in self.aoChildren: + aoRet.extend(oChild.getListOfLogFilesByKind(asKinds)); + + # Check our own files next. + for oFile in self.aoFiles: + if oFile.sKind in asKinds: + aoRet.append(oFile); + + return aoRet; + + def getFullName(self): + """ Constructs the full name of this test result. """ + if self.oParent is None: + return self.sName; + return self.oParent.getFullName() + ' / ' + self.sName; + + + +class TestResultValueData(ModelDataBase): + """ + Test result value data. + """ + + ksIdAttr = 'idTestResultValue'; + + ksParam_idTestResultValue = 'TestResultValue_idTestResultValue'; + ksParam_idTestResult = 'TestResultValue_idTestResult'; + ksParam_idTestSet = 'TestResultValue_idTestSet'; + ksParam_tsCreated = 'TestResultValue_tsCreated'; + ksParam_idStrName = 'TestResultValue_idStrName'; + ksParam_lValue = 'TestResultValue_lValue'; + ksParam_iUnit = 'TestResultValue_iUnit'; + + kasAllowNullAttributes = [ 'idTestSet', ]; + + def __init__(self): + ModelDataBase.__init__(self) + self.idTestResultValue = None; + self.idTestResult = None; + self.idTestSet = None; + self.tsCreated = None; + self.idStrName = None; + self.lValue = None; + self.iUnit = 0; + + def initFromDbRow(self, aoRow): + """ + Reinitialize from a SELECT * FROM TestResultValues. + Return self. Raises exception if no row. + """ + if aoRow is None: + raise TMRowNotFound('Test result value record not found.') + + self.idTestResultValue = aoRow[0]; + self.idTestResult = aoRow[1]; + self.idTestSet = aoRow[2]; + self.tsCreated = aoRow[3]; + self.idStrName = aoRow[4]; + self.lValue = aoRow[5]; + self.iUnit = aoRow[6]; + return self; + + +class TestResultValueDataEx(TestResultValueData): + """ + Extends TestResultValue by resolving the value name and unit string. + """ + + def __init__(self): + TestResultValueData.__init__(self) + self.sName = None; + self.sUnit = ''; + + def initFromDbRow(self, aoRow): + """ + Reinitialize from a query like this: + SELECT TestResultValues.*, TestResultStrTab.sValue + FROM TestResultValues, TestResultStrTab + WHERE TestResultStrTab.idStr = TestResultValues.idStrName + + Return self. Raises exception if no row. + """ + TestResultValueData.initFromDbRow(self, aoRow); + self.sName = aoRow[7]; + if self.iUnit < len(constants.valueunit.g_asNames): + self.sUnit = constants.valueunit.g_asNames[self.iUnit]; + else: + self.sUnit = '<%d>' % (self.iUnit,); + return self; + +class TestResultMsgData(ModelDataBase): + """ + Test result message data. + """ + + ksIdAttr = 'idTestResultMsg'; + + ksParam_idTestResultMsg = 'TestResultValue_idTestResultMsg'; + ksParam_idTestResult = 'TestResultValue_idTestResult'; + ksParam_idTestSet = 'TestResultValue_idTestSet'; + ksParam_tsCreated = 'TestResultValue_tsCreated'; + ksParam_idStrMsg = 'TestResultValue_idStrMsg'; + ksParam_enmLevel = 'TestResultValue_enmLevel'; + + kasAllowNullAttributes = [ 'idTestSet', ]; + + kcDbColumns = 6 + + def __init__(self): + ModelDataBase.__init__(self) + self.idTestResultMsg = None; + self.idTestResult = None; + self.idTestSet = None; + self.tsCreated = None; + self.idStrMsg = None; + self.enmLevel = None; + + def initFromDbRow(self, aoRow): + """ + Reinitialize from a SELECT * FROM TestResultMsgs. + Return self. Raises exception if no row. + """ + if aoRow is None: + raise TMRowNotFound('Test result value record not found.') + + self.idTestResultMsg = aoRow[0]; + self.idTestResult = aoRow[1]; + self.idTestSet = aoRow[2]; + self.tsCreated = aoRow[3]; + self.idStrMsg = aoRow[4]; + self.enmLevel = aoRow[5]; + return self; + +class TestResultMsgDataEx(TestResultMsgData): + """ + Extends TestResultMsg by resolving the message string. + """ + + def __init__(self): + TestResultMsgData.__init__(self) + self.sMsg = None; + + def initFromDbRow(self, aoRow): + """ + Reinitialize from a query like this: + SELECT TestResultMsg.*, TestResultStrTab.sValue + FROM TestResultMsg, TestResultStrTab + WHERE TestResultStrTab.idStr = TestResultMsgs.idStrName + + Return self. Raises exception if no row. + """ + TestResultMsgData.initFromDbRow(self, aoRow); + self.sMsg = aoRow[self.kcDbColumns]; + return self; + + +class TestResultFileData(ModelDataBase): + """ + Test result message data. + """ + + ksIdAttr = 'idTestResultFile'; + + ksParam_idTestResultFile = 'TestResultFile_idTestResultFile'; + ksParam_idTestResult = 'TestResultFile_idTestResult'; + ksParam_tsCreated = 'TestResultFile_tsCreated'; + ksParam_idStrFile = 'TestResultFile_idStrFile'; + ksParam_idStrDescription = 'TestResultFile_idStrDescription'; + ksParam_idStrKind = 'TestResultFile_idStrKind'; + ksParam_idStrMime = 'TestResultFile_idStrMime'; + + ## @name Kind of files. + ## @{ + ksKind_LogReleaseVm = 'log/release/vm'; + ksKind_LogDebugVm = 'log/debug/vm'; + ksKind_LogReleaseSvc = 'log/release/svc'; + ksKind_LogDebugSvc = 'log/debug/svc'; + ksKind_LogReleaseClient = 'log/release/client'; + ksKind_LogDebugClient = 'log/debug/client'; + ksKind_LogInstaller = 'log/installer'; + ksKind_LogUninstaller = 'log/uninstaller'; + ksKind_LogGuestKernel = 'log/guest/kernel'; + ksKind_ProcessReportVm = 'process/report/vm'; + ksKind_CrashReportVm = 'crash/report/vm'; + ksKind_CrashDumpVm = 'crash/dump/vm'; + ksKind_CrashReportSvc = 'crash/report/svc'; + ksKind_CrashDumpSvc = 'crash/dump/svc'; + ksKind_CrashReportClient = 'crash/report/client'; + ksKind_CrashDumpClient = 'crash/dump/client'; + ksKind_InfoCollection = 'info/collection'; + ksKind_InfoVgaText = 'info/vgatext'; + ksKind_MiscOther = 'misc/other'; + ksKind_ScreenshotFailure = 'screenshot/failure'; + ksKind_ScreenshotSuccesss = 'screenshot/success'; + ksKind_ScreenRecordingFailure = 'screenrecording/failure'; + ksKind_ScreenRecordingSuccess = 'screenrecording/success'; + ## @} + + kasKinds = [ + ksKind_LogReleaseVm, + ksKind_LogDebugVm, + ksKind_LogReleaseSvc, + ksKind_LogDebugSvc, + ksKind_LogReleaseClient, + ksKind_LogDebugClient, + ksKind_LogInstaller, + ksKind_LogUninstaller, + ksKind_LogGuestKernel, + ksKind_ProcessReportVm, + ksKind_CrashReportVm, + ksKind_CrashDumpVm, + ksKind_CrashReportSvc, + ksKind_CrashDumpSvc, + ksKind_CrashReportClient, + ksKind_CrashDumpClient, + ksKind_InfoCollection, + ksKind_InfoVgaText, + ksKind_MiscOther, + ksKind_ScreenshotFailure, + ksKind_ScreenshotSuccesss, + ksKind_ScreenRecordingFailure, + ksKind_ScreenRecordingSuccess, + ]; + + kasAllowNullAttributes = [ 'idTestSet', ]; + + kcDbColumns = 8 + + def __init__(self): + ModelDataBase.__init__(self) + self.idTestResultFile = None; + self.idTestResult = None; + self.idTestSet = None; + self.tsCreated = None; + self.idStrFile = None; + self.idStrDescription = None; + self.idStrKind = None; + self.idStrMime = None; + + def initFromDbRow(self, aoRow): + """ + Reinitialize from a SELECT * FROM TestResultFiles. + Return self. Raises exception if no row. + """ + if aoRow is None: + raise TMRowNotFound('Test result file record not found.') + + self.idTestResultFile = aoRow[0]; + self.idTestResult = aoRow[1]; + self.idTestSet = aoRow[2]; + self.tsCreated = aoRow[3]; + self.idStrFile = aoRow[4]; + self.idStrDescription = aoRow[5]; + self.idStrKind = aoRow[6]; + self.idStrMime = aoRow[7]; + return self; + +class TestResultFileDataEx(TestResultFileData): + """ + Extends TestResultFile by resolving the strings. + """ + + def __init__(self): + TestResultFileData.__init__(self) + self.sFile = None; + self.sDescription = None; + self.sKind = None; + self.sMime = None; + + def initFromDbRow(self, aoRow): + """ + Reinitialize from a query like this: + SELECT TestResultFiles.*, + StrTabFile.sValue AS sFile, + StrTabDesc.sValue AS sDescription + StrTabKind.sValue AS sKind, + StrTabMime.sValue AS sMime, + FROM ... + + Return self. Raises exception if no row. + """ + TestResultFileData.initFromDbRow(self, aoRow); + self.sFile = aoRow[self.kcDbColumns]; + self.sDescription = aoRow[self.kcDbColumns + 1]; + self.sKind = aoRow[self.kcDbColumns + 2]; + self.sMime = aoRow[self.kcDbColumns + 3]; + return self; + + def initFakeMainLog(self, oTestSet): + """ + Reinitializes to represent the main.log object (not in DB). + + Returns self. + """ + self.idTestResultFile = 0; + self.idTestResult = oTestSet.idTestResult; + self.tsCreated = oTestSet.tsCreated; + self.idStrFile = None; + self.idStrDescription = None; + self.idStrKind = None; + self.idStrMime = None; + + self.sFile = 'main.log'; + self.sDescription = ''; + self.sKind = 'log/main'; + self.sMime = 'text/plain'; + return self; + + def isProbablyUtf8Encoded(self): + """ + Checks if the file is likely to be UTF-8 encoded. + """ + if self.sMime in [ 'text/plain', 'text/html' ]: + return True; + return False; + + def getMimeWithEncoding(self): + """ + Gets the MIME type with encoding if likely to be UTF-8. + """ + if self.isProbablyUtf8Encoded(): + return '%s; charset=utf-8' % (self.sMime,); + return self.sMime; + + + +class TestResultListingData(ModelDataBase): # pylint: disable=too-many-instance-attributes + """ + Test case result data representation for table listing + """ + + class FailureReasonListingData(object): + """ Failure reason listing data """ + def __init__(self): + self.oFailureReason = None; + self.oFailureReasonAssigner = None; + self.tsFailureReasonAssigned = None; + self.sFailureReasonComment = None; + + def __init__(self): + """Initialize""" + ModelDataBase.__init__(self) + + self.idTestSet = None + + self.idBuildCategory = None; + self.sProduct = None + self.sRepository = None; + self.sBranch = None + self.sType = None + self.idBuild = None; + self.sVersion = None; + self.iRevision = None + + self.sOs = None; + self.sOsVersion = None; + self.sArch = None; + self.sCpuVendor = None; + self.sCpuName = None; + self.cCpus = None; + self.fCpuHwVirt = None; + self.fCpuNestedPaging = None; + self.fCpu64BitGuest = None; + self.idTestBox = None + self.sTestBoxName = None + + self.tsCreated = None + self.tsElapsed = None + self.enmStatus = None + self.cErrors = None; + + self.idTestCase = None + self.sTestCaseName = None + self.sBaseCmd = None + self.sArgs = None + self.sSubName = None; + + self.idBuildTestSuite = None; + self.iRevisionTestSuite = None; + + self.aoFailureReasons = []; + + def initFromDbRowEx(self, aoRow, oFailureReasonLogic, oUserAccountLogic): + """ + Reinitialize from a database query. + Return self. Raises exception if no row. + """ + if aoRow is None: + raise TMRowNotFound('Test result record not found.') + + self.idTestSet = aoRow[0]; + + self.idBuildCategory = aoRow[1]; + self.sProduct = aoRow[2]; + self.sRepository = aoRow[3]; + self.sBranch = aoRow[4]; + self.sType = aoRow[5]; + self.idBuild = aoRow[6]; + self.sVersion = aoRow[7]; + self.iRevision = aoRow[8]; + + self.sOs = aoRow[9]; + self.sOsVersion = aoRow[10]; + self.sArch = aoRow[11]; + self.sCpuVendor = aoRow[12]; + self.sCpuName = aoRow[13]; + self.cCpus = aoRow[14]; + self.fCpuHwVirt = aoRow[15]; + self.fCpuNestedPaging = aoRow[16]; + self.fCpu64BitGuest = aoRow[17]; + self.idTestBox = aoRow[18]; + self.sTestBoxName = aoRow[19]; + + self.tsCreated = aoRow[20]; + self.tsElapsed = aoRow[21]; + self.enmStatus = aoRow[22]; + self.cErrors = aoRow[23]; + + self.idTestCase = aoRow[24]; + self.sTestCaseName = aoRow[25]; + self.sBaseCmd = aoRow[26]; + self.sArgs = aoRow[27]; + self.sSubName = aoRow[28]; + + self.idBuildTestSuite = aoRow[29]; + self.iRevisionTestSuite = aoRow[30]; + + self.aoFailureReasons = []; + for i, _ in enumerate(aoRow[31]): + if aoRow[31][i] is not None \ + or aoRow[32][i] is not None \ + or aoRow[33][i] is not None \ + or aoRow[34][i] is not None: + oReason = self.FailureReasonListingData(); + if aoRow[31][i] is not None: + oReason.oFailureReason = oFailureReasonLogic.cachedLookup(aoRow[31][i]); + if aoRow[32][i] is not None: + oReason.oFailureReasonAssigner = oUserAccountLogic.cachedLookup(aoRow[32][i]); + oReason.tsFailureReasonAssigned = aoRow[33][i]; + oReason.sFailureReasonComment = aoRow[34][i]; + self.aoFailureReasons.append(oReason); + + return self + + +class TestResultHangingOffence(TMExceptionBase): + """Hanging offence committed by test case.""" + pass; # pylint: disable=unnecessary-pass + + +class TestResultFilter(ModelFilterBase): + """ + Test result filter. + """ + + kiTestStatus = 0; + kiErrorCounts = 1; + kiBranches = 2; + kiBuildTypes = 3; + kiRevisions = 4; + kiRevisionRange = 5; + kiFailReasons = 6; + kiTestCases = 7; + kiTestCaseMisc = 8; + kiTestBoxes = 9; + kiOses = 10; + kiCpuArches = 11; + kiCpuVendors = 12; + kiCpuCounts = 13; + kiMemory = 14; + kiTestboxMisc = 15; + kiPythonVersions = 16; + kiSchedGroups = 17; + + ## Misc test case / variation name filters. + ## Presented in table order. The first sub element is the presistent ID. + kaTcMisc = ( + ( 1, 'x86', ), + ( 2, 'amd64', ), + ( 3, 'uni', ), + ( 4, 'smp', ), + ( 5, 'raw', ), + ( 6, 'hw', ), + ( 7, 'np', ), + ( 8, 'Install', ), + ( 20, 'UInstall', ), # NB. out of order. + ( 9, 'Benchmark', ), + ( 18, 'smoke', ), # NB. out of order. + ( 19, 'unit', ), # NB. out of order. + ( 10, 'USB', ), + ( 11, 'Debian', ), + ( 12, 'Fedora', ), + ( 13, 'Oracle', ), + ( 14, 'RHEL', ), + ( 15, 'SUSE', ), + ( 16, 'Ubuntu', ), + ( 17, 'Win', ), + ); + + kiTbMisc_NestedPaging = 0; + kiTbMisc_NoNestedPaging = 1; + kiTbMisc_RawMode = 2; + kiTbMisc_NoRawMode = 3; + kiTbMisc_64BitGuest = 4; + kiTbMisc_No64BitGuest = 5; + kiTbMisc_HwVirt = 6; + kiTbMisc_NoHwVirt = 7; + kiTbMisc_IoMmu = 8; + kiTbMisc_NoIoMmu = 9; + + def __init__(self): + ModelFilterBase.__init__(self); + + # Test statuses + oCrit = FilterCriterion('Test statuses', sVarNm = 'ts', sType = FilterCriterion.ksType_String, + sTable = 'TestSets', sColumn = 'enmStatus'); + self.aCriteria.append(oCrit); + assert self.aCriteria[self.kiTestStatus] is oCrit; + + # Error counts + oCrit = FilterCriterion('Error counts', sVarNm = 'ec', sTable = 'TestResults', sColumn = 'cErrors'); + self.aCriteria.append(oCrit); + assert self.aCriteria[self.kiErrorCounts] is oCrit; + + # Branches + oCrit = FilterCriterion('Branches', sVarNm = 'br', sType = FilterCriterion.ksType_String, + sTable = 'BuildCategories', sColumn = 'sBranch'); + self.aCriteria.append(oCrit); + assert self.aCriteria[self.kiBranches] is oCrit; + + # Build types + oCrit = FilterCriterion('Build types', sVarNm = 'bt', sType = FilterCriterion.ksType_String, + sTable = 'BuildCategories', sColumn = 'sType'); + self.aCriteria.append(oCrit); + assert self.aCriteria[self.kiBuildTypes] is oCrit; + + # Revisions + oCrit = FilterCriterion('Revisions', sVarNm = 'rv', sTable = 'Builds', sColumn = 'iRevision'); + self.aCriteria.append(oCrit); + assert self.aCriteria[self.kiRevisions] is oCrit; + + # Revision Range + oCrit = FilterCriterion('Revision Range', sVarNm = 'rr', sType = FilterCriterion.ksType_Ranges, + sKind = FilterCriterion.ksKind_ElementOfOrNot, sTable = 'Builds', sColumn = 'iRevision'); + self.aCriteria.append(oCrit); + assert self.aCriteria[self.kiRevisionRange] is oCrit; + + # Failure reasons + oCrit = FilterCriterion('Failure reasons', sVarNm = 'fr', sType = FilterCriterion.ksType_UIntNil, + sTable = 'TestResultFailures', sColumn = 'idFailureReason'); + self.aCriteria.append(oCrit); + assert self.aCriteria[self.kiFailReasons] is oCrit; + + # Test cases and variations. + oCrit = FilterCriterion('Test case / var', sVarNm = 'tc', sTable = 'TestSets', sColumn = 'idTestCase', + oSub = FilterCriterion('Test variations', sVarNm = 'tv', + sTable = 'TestSets', sColumn = 'idTestCaseArgs')); + self.aCriteria.append(oCrit); + assert self.aCriteria[self.kiTestCases] is oCrit; + + # Special test case and varation name sub string matching. + oCrit = FilterCriterion('Test case name', sVarNm = 'cm', sKind = FilterCriterion.ksKind_Special, + asTables = ('TestCases', 'TestCaseArgs')); + oCrit.aoPossible = [ + FilterCriterionValueAndDescription(aoCur[0], 'Include %s' % (aoCur[1],)) for aoCur in self.kaTcMisc + ]; + oCrit.aoPossible.extend([ + FilterCriterionValueAndDescription(aoCur[0] + 32, 'Exclude %s' % (aoCur[1],)) for aoCur in self.kaTcMisc + ]); + self.aCriteria.append(oCrit); + assert self.aCriteria[self.kiTestCaseMisc] is oCrit; + + # Testboxes + oCrit = FilterCriterion('Testboxes', sVarNm = 'tb', sTable = 'TestSets', sColumn = 'idTestBox'); + self.aCriteria.append(oCrit); + assert self.aCriteria[self.kiTestBoxes] is oCrit; + + # Testbox OS and OS version. + oCrit = FilterCriterion('OS / version', sVarNm = 'os', sTable = 'TestBoxesWithStrings', sColumn = 'idStrOs', + oSub = FilterCriterion('OS Versions', sVarNm = 'ov', + sTable = 'TestBoxesWithStrings', sColumn = 'idStrOsVersion')); + self.aCriteria.append(oCrit); + assert self.aCriteria[self.kiOses] is oCrit; + + # Testbox CPU architectures. + oCrit = FilterCriterion('CPU arches', sVarNm = 'ca', sTable = 'TestBoxesWithStrings', sColumn = 'idStrCpuArch'); + self.aCriteria.append(oCrit); + assert self.aCriteria[self.kiCpuArches] is oCrit; + + # Testbox CPU vendors and revisions. + oCrit = FilterCriterion('CPU vendor / rev', sVarNm = 'cv', sTable = 'TestBoxesWithStrings', sColumn = 'idStrCpuVendor', + oSub = FilterCriterion('CPU revisions', sVarNm = 'cr', + sTable = 'TestBoxesWithStrings', sColumn = 'lCpuRevision')); + self.aCriteria.append(oCrit); + assert self.aCriteria[self.kiCpuVendors] is oCrit; + + # Testbox CPU (thread) count + oCrit = FilterCriterion('CPU counts', sVarNm = 'cc', sTable = 'TestBoxesWithStrings', sColumn = 'cCpus'); + self.aCriteria.append(oCrit); + assert self.aCriteria[self.kiCpuCounts] is oCrit; + + # Testbox memory sizes. + oCrit = FilterCriterion('Memory', sVarNm = 'mb', sTable = 'TestBoxesWithStrings', sColumn = 'cMbMemory'); + self.aCriteria.append(oCrit); + assert self.aCriteria[self.kiMemory] is oCrit; + + # Testbox features. + oCrit = FilterCriterion('Testbox features', sVarNm = 'tm', sKind = FilterCriterion.ksKind_Special, + sTable = 'TestBoxesWithStrings'); + oCrit.aoPossible = [ + FilterCriterionValueAndDescription(self.kiTbMisc_NestedPaging, "req nested paging"), + FilterCriterionValueAndDescription(self.kiTbMisc_NoNestedPaging, "w/o nested paging"), + #FilterCriterionValueAndDescription(self.kiTbMisc_RawMode, "req raw-mode"), - not implemented yet. + #FilterCriterionValueAndDescription(self.kiTbMisc_NoRawMode, "w/o raw-mode"), - not implemented yet. + FilterCriterionValueAndDescription(self.kiTbMisc_64BitGuest, "req 64-bit guests"), + FilterCriterionValueAndDescription(self.kiTbMisc_No64BitGuest, "w/o 64-bit guests"), + FilterCriterionValueAndDescription(self.kiTbMisc_HwVirt, "req VT-x / AMD-V"), + FilterCriterionValueAndDescription(self.kiTbMisc_NoHwVirt, "w/o VT-x / AMD-V"), + #FilterCriterionValueAndDescription(self.kiTbMisc_IoMmu, "req I/O MMU"), - not implemented yet. + #FilterCriterionValueAndDescription(self.kiTbMisc_NoIoMmu, "w/o I/O MMU"), - not implemented yet. + ]; + self.aCriteria.append(oCrit); + assert self.aCriteria[self.kiTestboxMisc] is oCrit; + + # Testbox python versions. + oCrit = FilterCriterion('Python', sVarNm = 'py', sTable = 'TestBoxesWithStrings', sColumn = 'iPythonHexVersion'); + self.aCriteria.append(oCrit); + assert self.aCriteria[self.kiPythonVersions] is oCrit; + + # Scheduling groups. + oCrit = FilterCriterion('Sched groups', sVarNm = 'sg', sTable = 'TestSets', sColumn = 'idSchedGroup'); + self.aCriteria.append(oCrit); + assert self.aCriteria[self.kiSchedGroups] is oCrit; + + + kdTbMiscConditions = { + kiTbMisc_NestedPaging: 'TestBoxesWithStrings.fCpuNestedPaging IS TRUE', + kiTbMisc_NoNestedPaging: 'TestBoxesWithStrings.fCpuNestedPaging IS FALSE', + kiTbMisc_RawMode: 'TestBoxesWithStrings.fRawMode IS TRUE', + kiTbMisc_NoRawMode: 'TestBoxesWithStrings.fRawMode IS NOT TRUE', + kiTbMisc_64BitGuest: 'TestBoxesWithStrings.fCpu64BitGuest IS TRUE', + kiTbMisc_No64BitGuest: 'TestBoxesWithStrings.fCpu64BitGuest IS FALSE', + kiTbMisc_HwVirt: 'TestBoxesWithStrings.fCpuHwVirt IS TRUE', + kiTbMisc_NoHwVirt: 'TestBoxesWithStrings.fCpuHwVirt IS FALSE', + kiTbMisc_IoMmu: 'TestBoxesWithStrings.fChipsetIoMmu IS TRUE', + kiTbMisc_NoIoMmu: 'TestBoxesWithStrings.fChipsetIoMmu IS FALSE', + }; + + def _getWhereWorker(self, iCrit, oCrit, sExtraIndent, iOmit): + """ Formats one - main or sub. """ + sQuery = ''; + if oCrit.sState == FilterCriterion.ksState_Selected and iCrit != iOmit: + if iCrit == self.kiTestCaseMisc: + for iValue, sLike in self.kaTcMisc: + if iValue in oCrit.aoSelected: sNot = ''; + elif iValue + 32 in oCrit.aoSelected: sNot = 'NOT '; + else: continue; + sQuery += '%s AND %s (' % (sExtraIndent, sNot,); + if len(sLike) <= 3: # do word matching for small substrings (hw, np, smp, uni, ++). + sQuery += 'TestCases.sName ~ \'.*\\y%s\\y.*\' ' \ + 'OR COALESCE(TestCaseArgs.sSubName, \'\') ~ \'.*\\y%s\\y.*\')\n' \ + % ( sLike, sLike,); + else: + sQuery += 'TestCases.sName LIKE \'%%%s%%\' ' \ + 'OR COALESCE(TestCaseArgs.sSubName, \'\') LIKE \'%%%s%%\')\n' \ + % ( sLike, sLike,); + elif iCrit == self.kiTestboxMisc: + dConditions = self.kdTbMiscConditions; + for iValue in oCrit.aoSelected: + if iValue in dConditions: + sQuery += '%s AND %s\n' % (sExtraIndent, dConditions[iValue],); + elif oCrit.sType == FilterCriterion.ksType_Ranges: + assert not oCrit.aoPossible; + if oCrit.aoSelected: + asConditions = []; + for tRange in oCrit.aoSelected: + if tRange[0] == tRange[1]: + asConditions.append('%s.%s = %s' % (oCrit.asTables[0], oCrit.sColumn, tRange[0])); + elif tRange[1] is None: # 9999- + asConditions.append('%s.%s >= %s' % (oCrit.asTables[0], oCrit.sColumn, tRange[0])); + elif tRange[0] is None: # -9999 + asConditions.append('%s.%s <= %s' % (oCrit.asTables[0], oCrit.sColumn, tRange[1])); + else: + asConditions.append('%s.%s BETWEEN %s AND %s' % (oCrit.asTables[0], oCrit.sColumn, + tRange[0], tRange[1])); + if not oCrit.fInverted: + sQuery += '%s AND (%s)\n' % (sExtraIndent, ' OR '.join(asConditions)); + else: + sQuery += '%s AND NOT (%s)\n' % (sExtraIndent, ' OR '.join(asConditions)); + else: + assert len(oCrit.asTables) == 1; + sQuery += '%s AND (' % (sExtraIndent,); + + if oCrit.sType != FilterCriterion.ksType_UIntNil or max(oCrit.aoSelected) != -1: + if iCrit == self.kiMemory: + sQuery += '(%s.%s / 1024)' % (oCrit.asTables[0], oCrit.sColumn,); + else: + sQuery += '%s.%s' % (oCrit.asTables[0], oCrit.sColumn,); + if not oCrit.fInverted: + sQuery += ' IN ('; + else: + sQuery += ' NOT IN ('; + if oCrit.sType == FilterCriterion.ksType_String: + sQuery += ', '.join('\'%s\'' % (sValue,) for sValue in oCrit.aoSelected) + ')'; + else: + sQuery += ', '.join(str(iValue) for iValue in oCrit.aoSelected if iValue != -1) + ')'; + + if oCrit.sType == FilterCriterion.ksType_UIntNil \ + and -1 in oCrit.aoSelected: + if sQuery[-1] != '(': sQuery += ' OR '; + sQuery += '%s.%s IS NULL' % (oCrit.asTables[0], oCrit.sColumn,); + + if iCrit == self.kiFailReasons: + if oCrit.fInverted: + sQuery += '%s OR TestResultFailures.idFailureReason IS NULL\n' % (sExtraIndent,); + else: + sQuery += '%s AND TestSets.enmStatus >= \'failure\'::TestStatus_T\n' % (sExtraIndent,); + sQuery += ')\n'; + if oCrit.oSub is not None: + sQuery += self._getWhereWorker(iCrit | (((iCrit >> 8) + 1) << 8), oCrit.oSub, sExtraIndent, iOmit); + return sQuery; + + def getWhereConditions(self, sExtraIndent = '', iOmit = -1): + """ + Construct the WHERE conditions for the filter, optionally omitting one + criterion. + """ + sQuery = ''; + for iCrit, oCrit in enumerate(self.aCriteria): + sQuery += self._getWhereWorker(iCrit, oCrit, sExtraIndent, iOmit); + return sQuery; + + def getTableJoins(self, sExtraIndent = '', iOmit = -1, dOmitTables = None): + """ + Construct the WHERE conditions for the filter, optionally omitting one + criterion. + """ + afDone = { 'TestSets': True, }; + if dOmitTables is not None: + afDone.update(dOmitTables); + + sQuery = ''; + for iCrit, oCrit in enumerate(self.aCriteria): + if oCrit.sState == FilterCriterion.ksState_Selected \ + and iCrit != iOmit: + for sTable in oCrit.asTables: + if sTable not in afDone: + afDone[sTable] = True; + if sTable == 'Builds': + sQuery += '%sINNER JOIN Builds\n' \ + '%s ON Builds.idBuild = TestSets.idBuild\n' \ + '%s AND Builds.tsExpire > TestSets.tsCreated\n' \ + '%s AND Builds.tsEffective <= TestSets.tsCreated\n' \ + % ( sExtraIndent, sExtraIndent, sExtraIndent, sExtraIndent, ); + elif sTable == 'BuildCategories': + sQuery += '%sINNER JOIN BuildCategories\n' \ + '%s ON BuildCategories.idBuildCategory = TestSets.idBuildCategory\n' \ + % ( sExtraIndent, sExtraIndent, ); + elif sTable == 'TestBoxesWithStrings': + sQuery += '%sLEFT OUTER JOIN TestBoxesWithStrings\n' \ + '%s ON TestBoxesWithStrings.idGenTestBox = TestSets.idGenTestBox\n' \ + % ( sExtraIndent, sExtraIndent, ); + elif sTable == 'TestCases': + sQuery += '%sINNER JOIN TestCases\n' \ + '%s ON TestCases.idGenTestCase = TestSets.idGenTestCase\n' \ + % ( sExtraIndent, sExtraIndent, ); + elif sTable == 'TestCaseArgs': + sQuery += '%sINNER JOIN TestCaseArgs\n' \ + '%s ON TestCaseArgs.idGenTestCaseArgs = TestSets.idGenTestCaseArgs\n' \ + % ( sExtraIndent, sExtraIndent, ); + elif sTable == 'TestResults': + sQuery += '%sINNER JOIN TestResults\n' \ + '%s ON TestResults.idTestResult = TestSets.idTestResult\n' \ + % ( sExtraIndent, sExtraIndent, ); + elif sTable == 'TestResultFailures': + sQuery += '%sLEFT OUTER JOIN TestResultFailures\n' \ + '%s ON TestResultFailures.idTestSet = TestSets.idTestSet\n' \ + '%s AND TestResultFailures.tsExpire = \'infinity\'::TIMESTAMP\n' \ + % ( sExtraIndent, sExtraIndent, sExtraIndent, ); + else: + assert False, sTable; + return sQuery; + + def isJoiningWithTable(self, sTable): + """ Checks whether getTableJoins already joins with TestResultFailures. """ + for oCrit in self.aCriteria: + if oCrit.sState == FilterCriterion.ksState_Selected and sTable in oCrit.asTables: + return True; + return False + + + +class TestResultLogic(ModelLogicBase): # pylint: disable=too-few-public-methods + """ + Results grouped by scheduling group. + """ + + # + # Result grinding for displaying in the WUI. + # + + ksResultsGroupingTypeNone = 'ResultsGroupingTypeNone'; + ksResultsGroupingTypeTestGroup = 'ResultsGroupingTypeTestGroup'; + ksResultsGroupingTypeBuildCat = 'ResultsGroupingTypeBuildCat'; + ksResultsGroupingTypeBuildRev = 'ResultsGroupingTypeBuildRev'; + ksResultsGroupingTypeTestBox = 'ResultsGroupingTypeTestBox'; + ksResultsGroupingTypeTestCase = 'ResultsGroupingTypeTestCase'; + ksResultsGroupingTypeOS = 'ResultsGroupingTypeOS'; + ksResultsGroupingTypeArch = 'ResultsGroupingTypeArch'; + ksResultsGroupingTypeSchedGroup = 'ResultsGroupingTypeSchedGroup'; + + ## @name Result sorting options. + ## @{ + ksResultsSortByRunningAndStart = 'ResultsSortByRunningAndStart'; ##< Default + ksResultsSortByBuildRevision = 'ResultsSortByBuildRevision'; + ksResultsSortByTestBoxName = 'ResultsSortByTestBoxName'; + ksResultsSortByTestBoxOs = 'ResultsSortByTestBoxOs'; + ksResultsSortByTestBoxOsVersion = 'ResultsSortByTestBoxOsVersion'; + ksResultsSortByTestBoxOsArch = 'ResultsSortByTestBoxOsArch'; + ksResultsSortByTestBoxArch = 'ResultsSortByTestBoxArch'; + ksResultsSortByTestBoxCpuVendor = 'ResultsSortByTestBoxCpuVendor'; + ksResultsSortByTestBoxCpuName = 'ResultsSortByTestBoxCpuName'; + ksResultsSortByTestBoxCpuRev = 'ResultsSortByTestBoxCpuRev'; + ksResultsSortByTestBoxCpuFeatures = 'ResultsSortByTestBoxCpuFeatures'; + ksResultsSortByTestCaseName = 'ResultsSortByTestCaseName'; + ksResultsSortByFailureReason = 'ResultsSortByFailureReason'; + kasResultsSortBy = { + ksResultsSortByRunningAndStart, + ksResultsSortByBuildRevision, + ksResultsSortByTestBoxName, + ksResultsSortByTestBoxOs, + ksResultsSortByTestBoxOsVersion, + ksResultsSortByTestBoxOsArch, + ksResultsSortByTestBoxArch, + ksResultsSortByTestBoxCpuVendor, + ksResultsSortByTestBoxCpuName, + ksResultsSortByTestBoxCpuRev, + ksResultsSortByTestBoxCpuFeatures, + ksResultsSortByTestCaseName, + ksResultsSortByFailureReason, + }; + ## Used by the WUI for generating the drop down. + kaasResultsSortByTitles = ( + ( ksResultsSortByRunningAndStart, 'Running & Start TS' ), + ( ksResultsSortByBuildRevision, 'Build Revision' ), + ( ksResultsSortByTestBoxName, 'TestBox Name' ), + ( ksResultsSortByTestBoxOs, 'O/S' ), + ( ksResultsSortByTestBoxOsVersion, 'O/S Version' ), + ( ksResultsSortByTestBoxOsArch, 'O/S & Architecture' ), + ( ksResultsSortByTestBoxArch, 'Architecture' ), + ( ksResultsSortByTestBoxCpuVendor, 'CPU Vendor' ), + ( ksResultsSortByTestBoxCpuName, 'CPU Vendor & Name' ), + ( ksResultsSortByTestBoxCpuRev, 'CPU Vendor & Revision' ), + ( ksResultsSortByTestBoxCpuFeatures, 'CPU Features' ), + ( ksResultsSortByTestCaseName, 'Test Case Name' ), + ( ksResultsSortByFailureReason, 'Failure Reason' ), + ); + ## @} + + ## Default sort by map. + kdResultSortByMap = { + ksResultsSortByRunningAndStart: ( (), None, None, '', '' ), + ksResultsSortByBuildRevision: ( + # Sorting tables. + ('Builds',), + # Sorting table join(s). + ' AND TestSets.idBuild = Builds.idBuild' + ' AND Builds.tsExpire >= TestSets.tsCreated' + ' AND Builds.tsEffective <= TestSets.tsCreated', + # Start of ORDER BY statement. + ' Builds.iRevision DESC', + # Extra columns to fetch for the above ORDER BY to work in a SELECT DISTINCT statement. + '', + # Columns for the GROUP BY + ''), + ksResultsSortByTestBoxName: ( + ('TestBoxes',), + ' AND TestSets.idGenTestBox = TestBoxes.idGenTestBox', + ' TestBoxes.sName DESC', + '', '' ), + ksResultsSortByTestBoxOsArch: ( + ('TestBoxesWithStrings',), + ' AND TestSets.idGenTestBox = TestBoxesWithStrings.idGenTestBox', + ' TestBoxesWithStrings.sOs, TestBoxesWithStrings.sCpuArch', + '', '' ), + ksResultsSortByTestBoxOs: ( + ('TestBoxesWithStrings',), + ' AND TestSets.idGenTestBox = TestBoxesWithStrings.idGenTestBox', + ' TestBoxesWithStrings.sOs', + '', '' ), + ksResultsSortByTestBoxOsVersion: ( + ('TestBoxesWithStrings',), + ' AND TestSets.idGenTestBox = TestBoxesWithStrings.idGenTestBox', + ' TestBoxesWithStrings.sOs, TestBoxesWithStrings.sOsVersion DESC', + '', '' ), + ksResultsSortByTestBoxArch: ( + ('TestBoxesWithStrings',), + ' AND TestSets.idGenTestBox = TestBoxesWithStrings.idGenTestBox', + ' TestBoxesWithStrings.sCpuArch', + '', '' ), + ksResultsSortByTestBoxCpuVendor: ( + ('TestBoxesWithStrings',), + ' AND TestSets.idGenTestBox = TestBoxesWithStrings.idGenTestBox', + ' TestBoxesWithStrings.sCpuVendor', + '', '' ), + ksResultsSortByTestBoxCpuName: ( + ('TestBoxesWithStrings',), + ' AND TestSets.idGenTestBox = TestBoxesWithStrings.idGenTestBox', + ' TestBoxesWithStrings.sCpuVendor, TestBoxesWithStrings.sCpuName', + '', '' ), + ksResultsSortByTestBoxCpuRev: ( + ('TestBoxesWithStrings',), + ' AND TestSets.idGenTestBox = TestBoxesWithStrings.idGenTestBox', + ' TestBoxesWithStrings.sCpuVendor, TestBoxesWithStrings.lCpuRevision DESC', + ', TestBoxesWithStrings.lCpuRevision', + ', TestBoxesWithStrings.lCpuRevision' ), + ksResultsSortByTestBoxCpuFeatures: ( + ('TestBoxes',), + ' AND TestSets.idGenTestBox = TestBoxes.idGenTestBox', + ' TestBoxes.fCpuHwVirt DESC, TestBoxes.fCpuNestedPaging DESC, TestBoxes.fCpu64BitGuest DESC, TestBoxes.cCpus DESC', + '', + '' ), + ksResultsSortByTestCaseName: ( + ('TestCases',), + ' AND TestSets.idGenTestCase = TestCases.idGenTestCase', + ' TestCases.sName', + '', '' ), + ksResultsSortByFailureReason: ( + (), '', + 'asSortByFailureReason ASC', + ', array_agg(FailureReasons.sShort ORDER BY TestResultFailures.idTestResult) AS asSortByFailureReason', + '' ), + }; + + kdResultGroupingMap = { + ksResultsGroupingTypeNone: ( + # Grouping tables; + (), + # Grouping field; + None, + # Grouping where addition. + None, + # Sort by overrides. + {}, + ), + ksResultsGroupingTypeTestGroup: ('', 'TestSets.idTestGroup', None, {},), + ksResultsGroupingTypeTestBox: ('', 'TestSets.idTestBox', None, {},), + ksResultsGroupingTypeTestCase: ('', 'TestSets.idTestCase', None, {},), + ksResultsGroupingTypeOS: ( + ('TestBoxes',), + 'TestBoxes.idStrOs', + ' AND TestBoxes.idGenTestBox = TestSets.idGenTestBox', + {}, + ), + ksResultsGroupingTypeArch: ( + ('TestBoxes',), + 'TestBoxes.idStrCpuArch', + ' AND TestBoxes.idGenTestBox = TestSets.idGenTestBox', + {}, + ), + ksResultsGroupingTypeBuildCat: ('', 'TestSets.idBuildCategory', None, {},), + ksResultsGroupingTypeBuildRev: ( + ('Builds',), + 'Builds.iRevision', + ' AND Builds.idBuild = TestSets.idBuild' + ' AND Builds.tsExpire > TestSets.tsCreated' + ' AND Builds.tsEffective <= TestSets.tsCreated', + { ksResultsSortByBuildRevision: ( (), None, ' Builds.iRevision DESC' ), } + ), + ksResultsGroupingTypeSchedGroup: ( '', 'TestSets.idSchedGroup', None, {},), + }; + + + def __init__(self, oDb): + ModelLogicBase.__init__(self, oDb) + self.oFailureReasonLogic = None; + self.oUserAccountLogic = None; + + def _getTimePeriodQueryPart(self, tsNow, sInterval, sExtraIndent = ''): + """ + Get part of SQL query responsible for SELECT data within + specified period of time. + """ + assert sInterval is not None; # too many rows. + + cMonthsMourningPeriod = 2; # Stop reminding everyone about testboxes after 2 months. (May also speed up the query.) + if tsNow is None: + sRet = '(TestSets.tsDone IS NULL OR TestSets.tsDone >= (CURRENT_TIMESTAMP - \'%s\'::interval))\n' \ + '%s AND TestSets.tsCreated >= (CURRENT_TIMESTAMP - \'%s\'::interval - \'%u months\'::interval)\n' \ + % ( sInterval, + sExtraIndent, sInterval, cMonthsMourningPeriod); + else: + sTsNow = '\'%s\'::TIMESTAMP' % (tsNow,); # It's actually a string already. duh. + sRet = 'TestSets.tsCreated <= %s\n' \ + '%s AND TestSets.tsCreated >= (%s - \'%s\'::interval - \'%u months\'::interval)\n' \ + '%s AND (TestSets.tsDone IS NULL OR TestSets.tsDone >= (%s - \'%s\'::interval))\n' \ + % ( sTsNow, + sExtraIndent, sTsNow, sInterval, cMonthsMourningPeriod, + sExtraIndent, sTsNow, sInterval ); + return sRet + + def fetchResultsForListing(self, iStart, cMaxRows, tsNow, sInterval, oFilter, enmResultSortBy, # pylint: disable=too-many-arguments + enmResultsGroupingType, iResultsGroupingValue, fOnlyFailures, fOnlyNeedingReason): + """ + Fetches TestResults table content. + + If @param enmResultsGroupingType and @param iResultsGroupingValue + are not None, then resulting (returned) list contains only records + that match specified @param enmResultsGroupingType. + + If @param enmResultsGroupingType is None, then + @param iResultsGroupingValue is ignored. + + Returns an array (list) of TestResultData items, empty list if none. + Raises exception on error. + """ + + _ = oFilter; + + # + # Get SQL query parameters + # + if enmResultsGroupingType is None or enmResultsGroupingType not in self.kdResultGroupingMap: + raise TMExceptionBase('Unknown grouping type'); + if enmResultSortBy is None or enmResultSortBy not in self.kasResultsSortBy: + raise TMExceptionBase('Unknown sorting'); + asGroupingTables, sGroupingField, sGroupingCondition, dSortOverrides = self.kdResultGroupingMap[enmResultsGroupingType]; + if enmResultSortBy in dSortOverrides: + asSortTables, sSortWhere, sSortOrderBy, sSortColumns, sSortGroupBy = dSortOverrides[enmResultSortBy]; + else: + asSortTables, sSortWhere, sSortOrderBy, sSortColumns, sSortGroupBy = self.kdResultSortByMap[enmResultSortBy]; + + # + # Construct the query. + # + sQuery = 'SELECT DISTINCT TestSets.idTestSet,\n' \ + ' BuildCategories.idBuildCategory,\n' \ + ' BuildCategories.sProduct,\n' \ + ' BuildCategories.sRepository,\n' \ + ' BuildCategories.sBranch,\n' \ + ' BuildCategories.sType,\n' \ + ' Builds.idBuild,\n' \ + ' Builds.sVersion,\n' \ + ' Builds.iRevision,\n' \ + ' TestBoxesWithStrings.sOs,\n' \ + ' TestBoxesWithStrings.sOsVersion,\n' \ + ' TestBoxesWithStrings.sCpuArch,\n' \ + ' TestBoxesWithStrings.sCpuVendor,\n' \ + ' TestBoxesWithStrings.sCpuName,\n' \ + ' TestBoxesWithStrings.cCpus,\n' \ + ' TestBoxesWithStrings.fCpuHwVirt,\n' \ + ' TestBoxesWithStrings.fCpuNestedPaging,\n' \ + ' TestBoxesWithStrings.fCpu64BitGuest,\n' \ + ' TestBoxesWithStrings.idTestBox,\n' \ + ' TestBoxesWithStrings.sName,\n' \ + ' TestResults.tsCreated,\n' \ + ' COALESCE(TestResults.tsElapsed, CURRENT_TIMESTAMP - TestResults.tsCreated) AS tsElapsedTestResult,\n' \ + ' TestSets.enmStatus,\n' \ + ' TestResults.cErrors,\n' \ + ' TestCases.idTestCase,\n' \ + ' TestCases.sName,\n' \ + ' TestCases.sBaseCmd,\n' \ + ' TestCaseArgs.sArgs,\n' \ + ' TestCaseArgs.sSubName,\n' \ + ' TestSuiteBits.idBuild AS idBuildTestSuite,\n' \ + ' TestSuiteBits.iRevision AS iRevisionTestSuite,\n' \ + ' array_agg(TestResultFailures.idFailureReason ORDER BY TestResultFailures.idTestResult),\n' \ + ' array_agg(TestResultFailures.uidAuthor ORDER BY TestResultFailures.idTestResult),\n' \ + ' array_agg(TestResultFailures.tsEffective ORDER BY TestResultFailures.idTestResult),\n' \ + ' array_agg(TestResultFailures.sComment ORDER BY TestResultFailures.idTestResult),\n' \ + ' (TestSets.tsDone IS NULL) SortRunningFirst' + sSortColumns + '\n' \ + 'FROM ( SELECT TestSets.idTestSet AS idTestSet,\n' \ + ' TestSets.tsDone AS tsDone,\n' \ + ' TestSets.tsCreated AS tsCreated,\n' \ + ' TestSets.enmStatus AS enmStatus,\n' \ + ' TestSets.idBuild AS idBuild,\n' \ + ' TestSets.idBuildTestSuite AS idBuildTestSuite,\n' \ + ' TestSets.idGenTestBox AS idGenTestBox,\n' \ + ' TestSets.idGenTestCase AS idGenTestCase,\n' \ + ' TestSets.idGenTestCaseArgs AS idGenTestCaseArgs\n' \ + ' FROM TestSets\n'; + sQuery += oFilter.getTableJoins(' '); + if fOnlyNeedingReason and not oFilter.isJoiningWithTable('TestResultFailures'): + sQuery += '\n' \ + ' LEFT OUTER JOIN TestResultFailures\n' \ + ' ON TestSets.idTestSet = TestResultFailures.idTestSet\n' \ + ' AND TestResultFailures.tsExpire = \'infinity\'::TIMESTAMP'; + for asTables in [asGroupingTables, asSortTables]: + for sTable in asTables: + if not oFilter.isJoiningWithTable(sTable): + sQuery = sQuery[:-1] + ',\n ' + sTable + '\n'; + + sQuery += ' WHERE ' + self._getTimePeriodQueryPart(tsNow, sInterval, ' ') + \ + oFilter.getWhereConditions(' '); + if fOnlyFailures or fOnlyNeedingReason: + sQuery += ' AND TestSets.enmStatus != \'success\'::TestStatus_T\n' \ + ' AND TestSets.enmStatus != \'running\'::TestStatus_T\n'; + if fOnlyNeedingReason: + sQuery += ' AND TestResultFailures.idTestSet IS NULL\n'; + if sGroupingField is not None: + sQuery += ' AND %s = %d\n' % (sGroupingField, iResultsGroupingValue,); + if sGroupingCondition is not None: + sQuery += sGroupingCondition.replace(' AND ', ' AND '); + if sSortWhere is not None: + sQuery += sSortWhere.replace(' AND ', ' AND '); + sQuery += ' ORDER BY '; + if sSortOrderBy is not None and sSortOrderBy.find('FailureReason') < 0: + sQuery += sSortOrderBy + ',\n '; + sQuery += '(TestSets.tsDone IS NULL) DESC, TestSets.idTestSet DESC\n' \ + ' LIMIT %s OFFSET %s\n' % (cMaxRows, iStart,); + + # Note! INNER JOIN TestBoxesWithStrings performs miserable compared to LEFT OUTER JOIN. Doesn't matter for the result + # because TestSets.idGenTestBox is a foreign key and unique in TestBoxes. So, let's do what ever is faster. + sQuery += ' ) AS TestSets\n' \ + ' LEFT OUTER JOIN TestBoxesWithStrings\n' \ + ' ON TestSets.idGenTestBox = TestBoxesWithStrings.idGenTestBox' \ + ' LEFT OUTER JOIN Builds AS TestSuiteBits\n' \ + ' ON TestSuiteBits.idBuild = TestSets.idBuildTestSuite\n' \ + ' AND TestSuiteBits.tsExpire > TestSets.tsCreated\n' \ + ' AND TestSuiteBits.tsEffective <= TestSets.tsCreated\n' \ + ' LEFT OUTER JOIN TestResultFailures\n' \ + ' ON TestSets.idTestSet = TestResultFailures.idTestSet\n' \ + ' AND TestResultFailures.tsExpire = \'infinity\'::TIMESTAMP'; + if sSortOrderBy is not None and sSortOrderBy.find('FailureReason') >= 0: + sQuery += '\n' \ + ' LEFT OUTER JOIN FailureReasons\n' \ + ' ON TestResultFailures.idFailureReason = FailureReasons.idFailureReason\n' \ + ' AND FailureReasons.tsExpire = \'infinity\'::TIMESTAMP'; + sQuery += ',\n' \ + ' BuildCategories,\n' \ + ' Builds,\n' \ + ' TestResults,\n' \ + ' TestCases,\n' \ + ' TestCaseArgs\n'; + sQuery += 'WHERE TestSets.idTestSet = TestResults.idTestSet\n' \ + ' AND TestResults.idTestResultParent is NULL\n' \ + ' AND TestSets.idBuild = Builds.idBuild\n' \ + ' AND Builds.tsExpire > TestSets.tsCreated\n' \ + ' AND Builds.tsEffective <= TestSets.tsCreated\n' \ + ' AND Builds.idBuildCategory = BuildCategories.idBuildCategory\n' \ + ' AND TestSets.idGenTestCase = TestCases.idGenTestCase\n' \ + ' AND TestSets.idGenTestCaseArgs = TestCaseArgs.idGenTestCaseArgs\n'; + sQuery += 'GROUP BY TestSets.idTestSet,\n' \ + ' BuildCategories.idBuildCategory,\n' \ + ' BuildCategories.sProduct,\n' \ + ' BuildCategories.sRepository,\n' \ + ' BuildCategories.sBranch,\n' \ + ' BuildCategories.sType,\n' \ + ' Builds.idBuild,\n' \ + ' Builds.sVersion,\n' \ + ' Builds.iRevision,\n' \ + ' TestBoxesWithStrings.sOs,\n' \ + ' TestBoxesWithStrings.sOsVersion,\n' \ + ' TestBoxesWithStrings.sCpuArch,\n' \ + ' TestBoxesWithStrings.sCpuVendor,\n' \ + ' TestBoxesWithStrings.sCpuName,\n' \ + ' TestBoxesWithStrings.cCpus,\n' \ + ' TestBoxesWithStrings.fCpuHwVirt,\n' \ + ' TestBoxesWithStrings.fCpuNestedPaging,\n' \ + ' TestBoxesWithStrings.fCpu64BitGuest,\n' \ + ' TestBoxesWithStrings.idTestBox,\n' \ + ' TestBoxesWithStrings.sName,\n' \ + ' TestResults.tsCreated,\n' \ + ' tsElapsedTestResult,\n' \ + ' TestSets.enmStatus,\n' \ + ' TestResults.cErrors,\n' \ + ' TestCases.idTestCase,\n' \ + ' TestCases.sName,\n' \ + ' TestCases.sBaseCmd,\n' \ + ' TestCaseArgs.sArgs,\n' \ + ' TestCaseArgs.sSubName,\n' \ + ' TestSuiteBits.idBuild,\n' \ + ' TestSuiteBits.iRevision,\n' \ + ' SortRunningFirst' + sSortGroupBy + '\n'; + sQuery += 'ORDER BY '; + if sSortOrderBy is not None: + sQuery += sSortOrderBy.replace('TestBoxes.', 'TestBoxesWithStrings.') + ',\n '; + sQuery += '(TestSets.tsDone IS NULL) DESC, TestSets.idTestSet DESC\n'; + + # + # Execute the query and return the wrapped results. + # + self._oDb.execute(sQuery); + + if self.oFailureReasonLogic is None: + self.oFailureReasonLogic = FailureReasonLogic(self._oDb); + if self.oUserAccountLogic is None: + self.oUserAccountLogic = UserAccountLogic(self._oDb); + + aoRows = []; + for aoRow in self._oDb.fetchAll(): + aoRows.append(TestResultListingData().initFromDbRowEx(aoRow, self.oFailureReasonLogic, self.oUserAccountLogic)); + + return aoRows + + + def fetchTimestampsForLogViewer(self, idTestSet): + """ + Returns an ordered list with all the test result timestamps, both start + and end. + + The log viewer create anchors in the log text so we can jump directly to + the log lines relevant for a test event. + """ + self._oDb.execute('(\n' + 'SELECT tsCreated\n' + 'FROM TestResults\n' + 'WHERE idTestSet = %s\n' + ') UNION (\n' + 'SELECT tsCreated + tsElapsed\n' + 'FROM TestResults\n' + 'WHERE idTestSet = %s\n' + ' AND tsElapsed IS NOT NULL\n' + ') UNION (\n' + 'SELECT TestResultFiles.tsCreated\n' + 'FROM TestResultFiles\n' + 'WHERE idTestSet = %s\n' + ') UNION (\n' + 'SELECT tsCreated\n' + 'FROM TestResultValues\n' + 'WHERE idTestSet = %s\n' + ') UNION (\n' + 'SELECT TestResultMsgs.tsCreated\n' + 'FROM TestResultMsgs\n' + 'WHERE idTestSet = %s\n' + ') ORDER by 1' + , ( idTestSet, idTestSet, idTestSet, idTestSet, idTestSet, )); + return [aoRow[0] for aoRow in self._oDb.fetchAll()]; + + + def getEntriesCount(self, tsNow, sInterval, oFilter, enmResultsGroupingType, iResultsGroupingValue, + fOnlyFailures, fOnlyNeedingReason): + """ + Get number of table records. + + If @param enmResultsGroupingType and @param iResultsGroupingValue + are not None, then we count only only those records + that match specified @param enmResultsGroupingType. + + If @param enmResultsGroupingType is None, then + @param iResultsGroupingValue is ignored. + """ + _ = oFilter; + + # + # Get SQL query parameters + # + if enmResultsGroupingType is None: + raise TMExceptionBase('Unknown grouping type') + + if enmResultsGroupingType not in self.kdResultGroupingMap: + raise TMExceptionBase('Unknown grouping type') + asGroupingTables, sGroupingField, sGroupingCondition, _ = self.kdResultGroupingMap[enmResultsGroupingType]; + + # + # Construct the query. + # + sQuery = 'SELECT COUNT(TestSets.idTestSet)\n' \ + 'FROM TestSets\n'; + sQuery += oFilter.getTableJoins(); + if fOnlyNeedingReason and not oFilter.isJoiningWithTable('TestResultFailures'): + sQuery += ' LEFT OUTER JOIN TestResultFailures\n' \ + ' ON TestSets.idTestSet = TestResultFailures.idTestSet\n' \ + ' AND TestResultFailures.tsExpire = \'infinity\'::TIMESTAMP\n'; + for sTable in asGroupingTables: + if not oFilter.isJoiningWithTable(sTable): + sQuery = sQuery[:-1] + ',\n ' + sTable + '\n'; + sQuery += 'WHERE ' + self._getTimePeriodQueryPart(tsNow, sInterval) + \ + oFilter.getWhereConditions(); + if fOnlyFailures or fOnlyNeedingReason: + sQuery += ' AND TestSets.enmStatus != \'success\'::TestStatus_T\n' \ + ' AND TestSets.enmStatus != \'running\'::TestStatus_T\n'; + if fOnlyNeedingReason: + sQuery += ' AND TestResultFailures.idTestSet IS NULL\n'; + if sGroupingField is not None: + sQuery += ' AND %s = %d\n' % (sGroupingField, iResultsGroupingValue,); + if sGroupingCondition is not None: + sQuery += sGroupingCondition.replace(' AND ', ' AND '); + + # + # Execute the query and return the result. + # + self._oDb.execute(sQuery) + return self._oDb.fetchOne()[0] + + def getTestGroups(self, tsNow, sPeriod): + """ + Get list of uniq TestGroupData objects which + found in all test results. + """ + + self._oDb.execute('SELECT DISTINCT TestGroups.*\n' + 'FROM TestGroups, TestSets\n' + 'WHERE TestSets.idTestGroup = TestGroups.idTestGroup\n' + ' AND TestGroups.tsExpire > TestSets.tsCreated\n' + ' AND TestGroups.tsEffective <= TestSets.tsCreated' + ' AND ' + self._getTimePeriodQueryPart(tsNow, sPeriod)) + aaoRows = self._oDb.fetchAll() + aoRet = [] + for aoRow in aaoRows: + aoRet.append(TestGroupData().initFromDbRow(aoRow)) + return aoRet + + def getBuilds(self, tsNow, sPeriod): + """ + Get list of uniq BuildDataEx objects which + found in all test results. + """ + + self._oDb.execute('SELECT DISTINCT Builds.*, BuildCategories.*\n' + 'FROM Builds, BuildCategories, TestSets\n' + 'WHERE TestSets.idBuild = Builds.idBuild\n' + ' AND Builds.idBuildCategory = BuildCategories.idBuildCategory\n' + ' AND Builds.tsExpire > TestSets.tsCreated\n' + ' AND Builds.tsEffective <= TestSets.tsCreated' + ' AND ' + self._getTimePeriodQueryPart(tsNow, sPeriod)) + aaoRows = self._oDb.fetchAll() + aoRet = [] + for aoRow in aaoRows: + aoRet.append(BuildDataEx().initFromDbRow(aoRow)) + return aoRet + + def getTestBoxes(self, tsNow, sPeriod): + """ + Get list of uniq TestBoxData objects which + found in all test results. + """ + # Note! INNER JOIN TestBoxesWithStrings performs miserable compared to LEFT OUTER JOIN. Doesn't matter for the result + # because TestSets.idGenTestBox is a foreign key and unique in TestBoxes. So, let's do what ever is faster. + self._oDb.execute('SELECT TestBoxesWithStrings.*\n' + 'FROM ( SELECT idTestBox AS idTestBox,\n' + ' MAX(idGenTestBox) AS idGenTestBox\n' + ' FROM TestSets\n' + ' WHERE ' + self._getTimePeriodQueryPart(tsNow, sPeriod, ' ') + + ' GROUP BY idTestBox\n' + ' ) AS TestBoxIDs\n' + ' LEFT OUTER JOIN TestBoxesWithStrings\n' + ' ON TestBoxesWithStrings.idGenTestBox = TestBoxIDs.idGenTestBox\n' + 'ORDER BY TestBoxesWithStrings.sName\n' ); + aoRet = [] + for aoRow in self._oDb.fetchAll(): + aoRet.append(TestBoxData().initFromDbRow(aoRow)); + return aoRet + + def getTestCases(self, tsNow, sPeriod): + """ + Get a list of unique TestCaseData objects which is appears in the test + specified result period. + """ + + # Using LEFT OUTER JOIN instead of INNER JOIN in case it performs better, doesn't matter for the result. + self._oDb.execute('SELECT TestCases.*\n' + 'FROM ( SELECT idTestCase AS idTestCase,\n' + ' MAX(idGenTestCase) AS idGenTestCase\n' + ' FROM TestSets\n' + ' WHERE ' + self._getTimePeriodQueryPart(tsNow, sPeriod, ' ') + + ' GROUP BY idTestCase\n' + ' ) AS TestCasesIDs\n' + ' LEFT OUTER JOIN TestCases ON TestCases.idGenTestCase = TestCasesIDs.idGenTestCase\n' + 'ORDER BY TestCases.sName\n' ); + + aoRet = []; + for aoRow in self._oDb.fetchAll(): + aoRet.append(TestCaseData().initFromDbRow(aoRow)); + return aoRet + + def getOSes(self, tsNow, sPeriod): + """ + Get a list of [idStrOs, sOs] tuples of the OSes that appears in the specified result period. + """ + + # Note! INNER JOIN TestBoxesWithStrings performs miserable compared to LEFT OUTER JOIN. Doesn't matter for the result + # because TestSets.idGenTestBox is a foreign key and unique in TestBoxes. So, let's do what ever is faster. + self._oDb.execute('SELECT DISTINCT TestBoxesWithStrings.idStrOs, TestBoxesWithStrings.sOs\n' + 'FROM ( SELECT idTestBox AS idTestBox,\n' + ' MAX(idGenTestBox) AS idGenTestBox\n' + ' FROM TestSets\n' + ' WHERE ' + self._getTimePeriodQueryPart(tsNow, sPeriod, ' ') + + ' GROUP BY idTestBox\n' + ' ) AS TestBoxIDs\n' + ' LEFT OUTER JOIN TestBoxesWithStrings\n' + ' ON TestBoxesWithStrings.idGenTestBox = TestBoxIDs.idGenTestBox\n' + 'ORDER BY TestBoxesWithStrings.sOs\n' ); + return self._oDb.fetchAll(); + + def getArchitectures(self, tsNow, sPeriod): + """ + Get a list of [idStrCpuArch, sCpuArch] tuples of the architecutres + that appears in the specified result period. + """ + + # Note! INNER JOIN TestBoxesWithStrings performs miserable compared to LEFT OUTER JOIN. Doesn't matter for the result + # because TestSets.idGenTestBox is a foreign key and unique in TestBoxes. So, let's do what ever is faster. + self._oDb.execute('SELECT DISTINCT TestBoxesWithStrings.idStrCpuArch, TestBoxesWithStrings.sCpuArch\n' + 'FROM ( SELECT idTestBox AS idTestBox,\n' + ' MAX(idGenTestBox) AS idGenTestBox\n' + ' FROM TestSets\n' + ' WHERE ' + self._getTimePeriodQueryPart(tsNow, sPeriod, ' ') + + ' GROUP BY idTestBox\n' + ' ) AS TestBoxIDs\n' + ' LEFT OUTER JOIN TestBoxesWithStrings\n' + ' ON TestBoxesWithStrings.idGenTestBox = TestBoxIDs.idGenTestBox\n' + 'ORDER BY TestBoxesWithStrings.sCpuArch\n' ); + return self._oDb.fetchAll(); + + def getBuildCategories(self, tsNow, sPeriod): + """ + Get a list of BuildCategoryData that appears in the specified result period. + """ + + self._oDb.execute('SELECT DISTINCT BuildCategories.*\n' + 'FROM ( SELECT DISTINCT idBuildCategory AS idBuildCategory\n' + ' FROM TestSets\n' + ' WHERE ' + self._getTimePeriodQueryPart(tsNow, sPeriod, ' ') + + ' ) AS BuildCategoryIDs\n' + ' LEFT OUTER JOIN BuildCategories\n' + ' ON BuildCategories.idBuildCategory = BuildCategoryIDs.idBuildCategory\n' + 'ORDER BY BuildCategories.sProduct, BuildCategories.sBranch, BuildCategories.sType\n'); + aoRet = []; + for aoRow in self._oDb.fetchAll(): + aoRet.append(BuildCategoryData().initFromDbRow(aoRow)); + return aoRet; + + def getSchedGroups(self, tsNow, sPeriod): + """ + Get list of uniq SchedGroupData objects which + found in all test results. + """ + + self._oDb.execute('SELECT SchedGroups.*\n' + 'FROM ( SELECT idSchedGroup,\n' + ' MAX(TestSets.tsCreated) AS tsNow\n' + ' FROM TestSets\n' + ' WHERE ' + self._getTimePeriodQueryPart(tsNow, sPeriod, ' ') + + ' GROUP BY idSchedGroup\n' + ' ) AS SchedGroupIDs\n' + ' INNER JOIN SchedGroups\n' + ' ON SchedGroups.idSchedGroup = SchedGroupIDs.idSchedGroup\n' + ' AND SchedGroups.tsExpire > SchedGroupIDs.tsNow\n' + ' AND SchedGroups.tsEffective <= SchedGroupIDs.tsNow\n' + 'ORDER BY SchedGroups.sName\n' ); + aoRet = [] + for aoRow in self._oDb.fetchAll(): + aoRet.append(SchedGroupData().initFromDbRow(aoRow)); + return aoRet + + def getById(self, idTestResult): + """ + Get build record by its id + """ + self._oDb.execute('SELECT *\n' + 'FROM TestResults\n' + 'WHERE idTestResult = %s\n', + (idTestResult,)) + + aRows = self._oDb.fetchAll() + if len(aRows) not in (0, 1): + raise TMTooManyRows('Found more than one test result with the same credentials. Database structure is corrupted.') + try: + return TestResultData().initFromDbRow(aRows[0]) + except IndexError: + return None + + def fetchPossibleFilterOptions(self, oFilter, tsNow, sPeriod, oReportModel = None): + """ + Fetches the available filter criteria, given the current filtering. + + Returns oFilter. + """ + assert isinstance(oFilter, TestResultFilter); + + # Hack to avoid lot's of conditionals or duplicate this code. + if oReportModel is None: + class DummyReportModel(object): + """ Dummy """ + def getExtraSubjectTables(self): + """ Dummy """ + return []; + def getExtraSubjectWhereExpr(self): + """ Dummy """ + return ''; + oReportModel = DummyReportModel(); + + def workerDoFetch(oMissingLogicType, sNameAttr = 'sName', fIdIsName = False, idxHover = -1, + idNull = -1, sNullDesc = '<NULL>'): + """ Does the tedious result fetching and handling of missing bits. """ + dLeft = { oValue: 1 for oValue in oCrit.aoSelected }; + oCrit.aoPossible = []; + for aoRow in self._oDb.fetchAll(): + oCrit.aoPossible.append(FilterCriterionValueAndDescription(aoRow[0] if aoRow[0] is not None else idNull, + aoRow[1] if aoRow[1] is not None else sNullDesc, + aoRow[2], + aoRow[idxHover] if idxHover >= 0 else None)); + if aoRow[0] in dLeft: + del dLeft[aoRow[0]]; + if dLeft: + if fIdIsName: + for idMissing in dLeft: + oCrit.aoPossible.append(FilterCriterionValueAndDescription(idMissing, str(idMissing), + fIrrelevant = True)); + else: + oMissingLogic = oMissingLogicType(self._oDb); + for idMissing in dLeft: + oMissing = oMissingLogic.cachedLookup(idMissing); + if oMissing is not None: + oCrit.aoPossible.append(FilterCriterionValueAndDescription(idMissing, + getattr(oMissing, sNameAttr), + fIrrelevant = True)); + + def workerDoFetchNested(): + """ Does the tedious result fetching and handling of missing bits. """ + oCrit.aoPossible = []; + oCrit.oSub.aoPossible = []; + dLeft = { oValue: 1 for oValue in oCrit.aoSelected }; + dSubLeft = { oValue: 1 for oValue in oCrit.oSub.aoSelected }; + oMain = None; + for aoRow in self._oDb.fetchAll(): + if oMain is None or oMain.oValue != aoRow[0]: + oMain = FilterCriterionValueAndDescription(aoRow[0], aoRow[1], 0); + oCrit.aoPossible.append(oMain); + if aoRow[0] in dLeft: + del dLeft[aoRow[0]]; + oCurSub = FilterCriterionValueAndDescription(aoRow[2], aoRow[3], aoRow[4]); + oCrit.oSub.aoPossible.append(oCurSub); + if aoRow[2] in dSubLeft: + del dSubLeft[aoRow[2]]; + + oMain.aoSubs.append(oCurSub); + oMain.cTimes += aoRow[4]; + + if dLeft: + pass; ## @todo + + # Statuses. + oCrit = oFilter.aCriteria[TestResultFilter.kiTestStatus]; + self._oDb.execute('SELECT TestSets.enmStatus, TestSets.enmStatus, COUNT(TestSets.idTestSet)\n' + 'FROM TestSets\n' + oFilter.getTableJoins(iOmit = TestResultFilter.kiTestStatus) + + ''.join(' , %s\n' % (sTable,) for sTable in oReportModel.getExtraSubjectTables()) + + 'WHERE ' + self._getTimePeriodQueryPart(tsNow, sPeriod) + + oFilter.getWhereConditions(iOmit = TestResultFilter.kiTestStatus) + + oReportModel.getExtraSubjectWhereExpr() + + 'GROUP BY TestSets.enmStatus\n' + 'ORDER BY TestSets.enmStatus\n'); + workerDoFetch(None, fIdIsName = True); + + # Scheduling groups (see getSchedGroups). + oCrit = oFilter.aCriteria[TestResultFilter.kiSchedGroups]; + self._oDb.execute('SELECT SchedGroups.idSchedGroup, SchedGroups.sName, SchedGroupIDs.cTimes\n' + 'FROM ( SELECT TestSets.idSchedGroup,\n' + ' MAX(TestSets.tsCreated) AS tsNow,\n' + ' COUNT(TestSets.idTestSet) AS cTimes\n' + ' FROM TestSets\n' + oFilter.getTableJoins(iOmit = TestResultFilter.kiSchedGroups) + + ''.join(' , %s\n' % (sTable,) for sTable in oReportModel.getExtraSubjectTables()) + + ' WHERE ' + self._getTimePeriodQueryPart(tsNow, sPeriod, ' ') + + oFilter.getWhereConditions(iOmit = TestResultFilter.kiSchedGroups) + + oReportModel.getExtraSubjectWhereExpr() + + ' GROUP BY TestSets.idSchedGroup\n' + ' ) AS SchedGroupIDs\n' + ' INNER JOIN SchedGroups\n' + ' ON SchedGroups.idSchedGroup = SchedGroupIDs.idSchedGroup\n' + ' AND SchedGroups.tsExpire > SchedGroupIDs.tsNow\n' + ' AND SchedGroups.tsEffective <= SchedGroupIDs.tsNow\n' + 'ORDER BY SchedGroups.sName\n' ); + workerDoFetch(SchedGroupLogic); + + # Testboxes (see getTestBoxes). + oCrit = oFilter.aCriteria[TestResultFilter.kiTestBoxes]; + self._oDb.execute('SELECT TestBoxesWithStrings.idTestBox,\n' + ' TestBoxesWithStrings.sName,\n' + ' TestBoxIDs.cTimes\n' + 'FROM ( SELECT TestSets.idTestBox AS idTestBox,\n' + ' MAX(TestSets.idGenTestBox) AS idGenTestBox,\n' + ' COUNT(TestSets.idTestSet) AS cTimes\n' + ' FROM TestSets\n' + oFilter.getTableJoins(iOmit = TestResultFilter.kiTestBoxes) + + ''.join(' , %s\n' % (sTable,) for sTable in oReportModel.getExtraSubjectTables()) + + ' WHERE ' + self._getTimePeriodQueryPart(tsNow, sPeriod, ' ') + + oFilter.getWhereConditions(iOmit = TestResultFilter.kiTestBoxes) + + oReportModel.getExtraSubjectWhereExpr() + + ' GROUP BY TestSets.idTestBox\n' + ' ) AS TestBoxIDs\n' + ' LEFT OUTER JOIN TestBoxesWithStrings\n' + ' ON TestBoxesWithStrings.idGenTestBox = TestBoxIDs.idGenTestBox\n' + 'ORDER BY TestBoxesWithStrings.sName\n' ); + workerDoFetch(TestBoxLogic); + + # Testbox OSes and versions. + oCrit = oFilter.aCriteria[TestResultFilter.kiOses]; + self._oDb.execute('SELECT TestBoxesWithStrings.idStrOs,\n' + ' TestBoxesWithStrings.sOs,\n' + ' TestBoxesWithStrings.idStrOsVersion,\n' + ' TestBoxesWithStrings.sOsVersion,\n' + ' SUM(TestBoxGenIDs.cTimes)\n' + 'FROM ( SELECT TestSets.idGenTestBox,\n' + ' COUNT(TestSets.idTestSet) AS cTimes\n' + ' FROM TestSets\n' + oFilter.getTableJoins(iOmit = TestResultFilter.kiOses) + + ''.join(' , %s\n' % (sTable,) for sTable in oReportModel.getExtraSubjectTables()) + + ' WHERE ' + self._getTimePeriodQueryPart(tsNow, sPeriod, ' ') + + oFilter.getWhereConditions(iOmit = TestResultFilter.kiOses) + + oReportModel.getExtraSubjectWhereExpr() + + ' GROUP BY TestSets.idGenTestBox\n' + ' ) AS TestBoxGenIDs\n' + ' LEFT OUTER JOIN TestBoxesWithStrings\n' + ' ON TestBoxesWithStrings.idGenTestBox = TestBoxGenIDs.idGenTestBox\n' + 'GROUP BY TestBoxesWithStrings.idStrOs,\n' + ' TestBoxesWithStrings.sOs,\n' + ' TestBoxesWithStrings.idStrOsVersion,\n' + ' TestBoxesWithStrings.sOsVersion\n' + 'ORDER BY TestBoxesWithStrings.sOs,\n' + ' TestBoxesWithStrings.sOs = \'win\' AND TestBoxesWithStrings.sOsVersion = \'10\' DESC,\n' + ' TestBoxesWithStrings.sOsVersion DESC\n' + ); + workerDoFetchNested(); + + # Testbox CPU(/OS) architectures. + oCrit = oFilter.aCriteria[TestResultFilter.kiCpuArches]; + self._oDb.execute('SELECT TestBoxesWithStrings.idStrCpuArch,\n' + ' TestBoxesWithStrings.sCpuArch,\n' + ' SUM(TestBoxGenIDs.cTimes)\n' + 'FROM ( SELECT TestSets.idGenTestBox,\n' + ' COUNT(TestSets.idTestSet) AS cTimes\n' + ' FROM TestSets\n' + oFilter.getTableJoins(iOmit = TestResultFilter.kiCpuArches) + + ''.join(' , %s\n' % (sTable,) for sTable in oReportModel.getExtraSubjectTables()) + + ' WHERE ' + self._getTimePeriodQueryPart(tsNow, sPeriod, ' ') + + oFilter.getWhereConditions(iOmit = TestResultFilter.kiCpuArches) + + oReportModel.getExtraSubjectWhereExpr() + + ' GROUP BY TestSets.idGenTestBox\n' + ' ) AS TestBoxGenIDs\n' + ' LEFT OUTER JOIN TestBoxesWithStrings\n' + ' ON TestBoxesWithStrings.idGenTestBox = TestBoxGenIDs.idGenTestBox\n' + 'GROUP BY TestBoxesWithStrings.idStrCpuArch, TestBoxesWithStrings.sCpuArch\n' + 'ORDER BY TestBoxesWithStrings.sCpuArch\n' ); + workerDoFetch(None, fIdIsName = True); + + # Testbox CPU revisions. + oCrit = oFilter.aCriteria[TestResultFilter.kiCpuVendors]; + self._oDb.execute('SELECT TestBoxesWithStrings.idStrCpuVendor,\n' + ' TestBoxesWithStrings.sCpuVendor,\n' + ' TestBoxesWithStrings.lCpuRevision,\n' + ' TestBoxesWithStrings.sCpuVendor,\n' + ' SUM(TestBoxGenIDs.cTimes)\n' + 'FROM ( SELECT TestSets.idGenTestBox,\n' + ' COUNT(TestSets.idTestSet) AS cTimes\n' + ' FROM TestSets\n' + oFilter.getTableJoins(iOmit = TestResultFilter.kiCpuVendors) + + ''.join(' , %s\n' % (sTable,) for sTable in oReportModel.getExtraSubjectTables()) + + ' WHERE ' + self._getTimePeriodQueryPart(tsNow, sPeriod, ' ') + + oFilter.getWhereConditions(iOmit = TestResultFilter.kiCpuVendors) + + oReportModel.getExtraSubjectWhereExpr() + + ' GROUP BY TestSets.idGenTestBox' + ' ) AS TestBoxGenIDs\n' + ' LEFT OUTER JOIN TestBoxesWithStrings\n' + ' ON TestBoxesWithStrings.idGenTestBox = TestBoxGenIDs.idGenTestBox\n' + 'GROUP BY TestBoxesWithStrings.idStrCpuVendor,\n' + ' TestBoxesWithStrings.sCpuVendor,\n' + ' TestBoxesWithStrings.lCpuRevision,\n' + ' TestBoxesWithStrings.sCpuVendor\n' + 'ORDER BY TestBoxesWithStrings.sCpuVendor DESC,\n' + ' TestBoxesWithStrings.sCpuVendor = \'GenuineIntel\'\n' + ' AND (TestBoxesWithStrings.lCpuRevision >> 24) = 15,\n' # P4 at the bottom is a start... + ' TestBoxesWithStrings.lCpuRevision DESC\n' + ); + workerDoFetchNested(); + for oCur in oCrit.oSub.aoPossible: + oCur.sDesc = TestBoxData.getPrettyCpuVersionEx(oCur.oValue, oCur.sDesc).replace('_', ' '); + + # Testbox CPU core/thread counts. + oCrit = oFilter.aCriteria[TestResultFilter.kiCpuCounts]; + self._oDb.execute('SELECT TestBoxesWithStrings.cCpus,\n' + ' CAST(TestBoxesWithStrings.cCpus AS TEXT),\n' + ' SUM(TestBoxGenIDs.cTimes)\n' + 'FROM ( SELECT TestSets.idGenTestBox,\n' + ' COUNT(TestSets.idTestSet) AS cTimes\n' + ' FROM TestSets\n' + oFilter.getTableJoins(iOmit = TestResultFilter.kiCpuCounts) + + ''.join(' , %s\n' % (sTable,) for sTable in oReportModel.getExtraSubjectTables()) + + ' WHERE ' + self._getTimePeriodQueryPart(tsNow, sPeriod, ' ') + + oFilter.getWhereConditions(iOmit = TestResultFilter.kiCpuCounts) + + oReportModel.getExtraSubjectWhereExpr() + + ' GROUP BY TestSets.idGenTestBox' + ' ) AS TestBoxGenIDs\n' + ' LEFT OUTER JOIN TestBoxesWithStrings\n' + ' ON TestBoxesWithStrings.idGenTestBox = TestBoxGenIDs.idGenTestBox\n' + 'GROUP BY TestBoxesWithStrings.cCpus\n' + 'ORDER BY TestBoxesWithStrings.cCpus\n' ); + workerDoFetch(None, fIdIsName = True); + + # Testbox memory. + oCrit = oFilter.aCriteria[TestResultFilter.kiMemory]; + self._oDb.execute('SELECT TestBoxesWithStrings.cMbMemory / 1024,\n' + ' NULL,\n' + ' SUM(TestBoxGenIDs.cTimes)\n' + 'FROM ( SELECT TestSets.idGenTestBox,\n' + ' COUNT(TestSets.idTestSet) AS cTimes\n' + ' FROM TestSets\n' + oFilter.getTableJoins(iOmit = TestResultFilter.kiMemory) + + ''.join(' , %s\n' % (sTable,) for sTable in oReportModel.getExtraSubjectTables()) + + ' WHERE ' + self._getTimePeriodQueryPart(tsNow, sPeriod, ' ') + + oFilter.getWhereConditions(iOmit = TestResultFilter.kiMemory) + + oReportModel.getExtraSubjectWhereExpr() + + ' GROUP BY TestSets.idGenTestBox' + ' ) AS TestBoxGenIDs\n' + ' LEFT OUTER JOIN TestBoxesWithStrings\n' + ' ON TestBoxesWithStrings.idGenTestBox = TestBoxGenIDs.idGenTestBox\n' + 'GROUP BY TestBoxesWithStrings.cMbMemory / 1024\n' + 'ORDER BY 1\n' ); + workerDoFetch(None, fIdIsName = True); + for oCur in oCrit.aoPossible: + oCur.sDesc = '%u GB' % (oCur.oValue,); + + # Testbox python versions . + oCrit = oFilter.aCriteria[TestResultFilter.kiPythonVersions]; + self._oDb.execute('SELECT TestBoxesWithStrings.iPythonHexVersion,\n' + ' NULL,\n' + ' SUM(TestBoxGenIDs.cTimes)\n' + 'FROM ( SELECT TestSets.idGenTestBox AS idGenTestBox,\n' + ' COUNT(TestSets.idTestSet) AS cTimes\n' + ' FROM TestSets\n' + oFilter.getTableJoins(iOmit = TestResultFilter.kiPythonVersions) + + ''.join(' , %s\n' % (sTable,) for sTable in oReportModel.getExtraSubjectTables()) + + ' WHERE ' + self._getTimePeriodQueryPart(tsNow, sPeriod, ' ') + + oFilter.getWhereConditions(iOmit = TestResultFilter.kiPythonVersions) + + oReportModel.getExtraSubjectWhereExpr() + + ' GROUP BY TestSets.idGenTestBox\n' + ' ) AS TestBoxGenIDs\n' + ' LEFT OUTER JOIN TestBoxesWithStrings\n' + ' ON TestBoxesWithStrings.idGenTestBox = TestBoxGenIDs.idGenTestBox\n' + 'GROUP BY TestBoxesWithStrings.iPythonHexVersion\n' + 'ORDER BY TestBoxesWithStrings.iPythonHexVersion\n' ); + workerDoFetch(None, fIdIsName = True); + for oCur in oCrit.aoPossible: + oCur.sDesc = TestBoxData.formatPythonVersionEx(oCur.oValue); # pylint: disable=redefined-variable-type + + # Testcase with variation. + oCrit = oFilter.aCriteria[TestResultFilter.kiTestCases]; + self._oDb.execute('SELECT TestCaseArgsIDs.idTestCase,\n' + ' TestCases.sName,\n' + ' TestCaseArgsIDs.idTestCaseArgs,\n' + ' CASE WHEN TestCaseArgs.sSubName IS NULL OR TestCaseArgs.sSubName = \'\' THEN\n' + ' CONCAT(\'/ #\', TestCaseArgs.idTestCaseArgs)\n' + ' ELSE\n' + ' TestCaseArgs.sSubName\n' + ' END,' + ' TestCaseArgsIDs.cTimes\n' + 'FROM ( SELECT TestSets.idTestCase AS idTestCase,\n' + ' TestSets.idTestCaseArgs AS idTestCaseArgs,\n' + ' MAX(TestSets.idGenTestCase) AS idGenTestCase,\n' + ' MAX(TestSets.idGenTestCaseArgs) AS idGenTestCaseArgs,\n' + ' COUNT(TestSets.idTestSet) AS cTimes\n' + ' FROM TestSets\n' + oFilter.getTableJoins(iOmit = TestResultFilter.kiTestCases) + + ''.join(' , %s\n' % (sTable,) for sTable in oReportModel.getExtraSubjectTables()) + + ' WHERE ' + self._getTimePeriodQueryPart(tsNow, sPeriod, ' ') + + oFilter.getWhereConditions(iOmit = TestResultFilter.kiTestCases) + + oReportModel.getExtraSubjectWhereExpr() + + ' GROUP BY TestSets.idTestCase, TestSets.idTestCaseArgs\n' + ' ) AS TestCaseArgsIDs\n' + ' LEFT OUTER JOIN TestCases ON TestCases.idGenTestCase = TestCaseArgsIDs.idGenTestCase\n' + ' LEFT OUTER JOIN TestCaseArgs\n' + ' ON TestCaseArgs.idGenTestCaseArgs = TestCaseArgsIDs.idGenTestCaseArgs\n' + 'ORDER BY TestCases.sName, 4\n' ); + workerDoFetchNested(); + + # Build revisions. + oCrit = oFilter.aCriteria[TestResultFilter.kiRevisions]; + self._oDb.execute('SELECT Builds.iRevision, CONCAT(\'r\', Builds.iRevision), SUM(BuildIDs.cTimes)\n' + 'FROM ( SELECT TestSets.idBuild AS idBuild,\n' + ' MAX(TestSets.tsCreated) AS tsNow,\n' + ' COUNT(TestSets.idBuild) AS cTimes\n' + ' FROM TestSets\n' + oFilter.getTableJoins(iOmit = TestResultFilter.kiRevisions) + + ''.join(' , %s\n' % (sTable,) for sTable in oReportModel.getExtraSubjectTables()) + + ' WHERE ' + self._getTimePeriodQueryPart(tsNow, sPeriod, ' ') + + oFilter.getWhereConditions(iOmit = TestResultFilter.kiRevisions) + + oReportModel.getExtraSubjectWhereExpr() + + ' GROUP BY TestSets.idBuild\n' + ' ) AS BuildIDs\n' + ' INNER JOIN Builds\n' + ' ON Builds.idBuild = BuildIDs.idBuild\n' + ' AND Builds.tsExpire > BuildIDs.tsNow\n' + ' AND Builds.tsEffective <= BuildIDs.tsNow\n' + 'GROUP BY Builds.iRevision\n' + 'ORDER BY Builds.iRevision DESC\n' ); + workerDoFetch(None, fIdIsName = True); + + # Build branches. + oCrit = oFilter.aCriteria[TestResultFilter.kiBranches]; + self._oDb.execute('SELECT BuildCategories.sBranch, BuildCategories.sBranch, SUM(BuildCategoryIDs.cTimes)\n' + 'FROM ( SELECT TestSets.idBuildCategory,\n' + ' COUNT(TestSets.idTestSet) AS cTimes\n' + ' FROM TestSets\n' + oFilter.getTableJoins(iOmit = TestResultFilter.kiBranches) + + ''.join(' , %s\n' % (sTable,) for sTable in oReportModel.getExtraSubjectTables()) + + ' WHERE ' + self._getTimePeriodQueryPart(tsNow, sPeriod, ' ') + + oFilter.getWhereConditions(iOmit = TestResultFilter.kiBranches) + + oReportModel.getExtraSubjectWhereExpr() + + ' GROUP BY TestSets.idBuildCategory\n' + ' ) AS BuildCategoryIDs\n' + ' INNER JOIN BuildCategories\n' + ' ON BuildCategories.idBuildCategory = BuildCategoryIDs.idBuildCategory\n' + 'GROUP BY BuildCategories.sBranch\n' + 'ORDER BY BuildCategories.sBranch DESC\n' ); + workerDoFetch(None, fIdIsName = True); + + # Build types. + oCrit = oFilter.aCriteria[TestResultFilter.kiBuildTypes]; + self._oDb.execute('SELECT BuildCategories.sType, BuildCategories.sType, SUM(BuildCategoryIDs.cTimes)\n' + 'FROM ( SELECT TestSets.idBuildCategory,\n' + ' COUNT(TestSets.idTestSet) AS cTimes\n' + ' FROM TestSets\n' + oFilter.getTableJoins(iOmit = TestResultFilter.kiBuildTypes) + + ''.join(' , %s\n' % (sTable,) for sTable in oReportModel.getExtraSubjectTables()) + + ' WHERE ' + self._getTimePeriodQueryPart(tsNow, sPeriod, ' ') + + oFilter.getWhereConditions(iOmit = TestResultFilter.kiBuildTypes) + + oReportModel.getExtraSubjectWhereExpr() + + ' GROUP BY TestSets.idBuildCategory\n' + ' ) AS BuildCategoryIDs\n' + ' INNER JOIN BuildCategories\n' + ' ON BuildCategories.idBuildCategory = BuildCategoryIDs.idBuildCategory\n' + 'GROUP BY BuildCategories.sType\n' + 'ORDER BY BuildCategories.sType DESC\n' ); + workerDoFetch(None, fIdIsName = True); + + # Failure reasons. + oCrit = oFilter.aCriteria[TestResultFilter.kiFailReasons]; + self._oDb.execute('SELECT FailureReasons.idFailureReason, FailureReasons.sShort, FailureReasonIDs.cTimes\n' + 'FROM ( SELECT TestResultFailures.idFailureReason,\n' + ' COUNT(TestSets.idTestSet) as cTimes\n' + ' FROM TestSets\n' + ' LEFT OUTER JOIN TestResultFailures\n' + ' ON TestResultFailures.idTestSet = TestSets.idTestSet\n' + ' AND TestResultFailures.tsExpire = \'infinity\'::TIMESTAMP\n' + + oFilter.getTableJoins(iOmit = TestResultFilter.kiFailReasons) + + ''.join(' , %s\n' % (sTable,) for sTable in oReportModel.getExtraSubjectTables()) + + ' WHERE ' + self._getTimePeriodQueryPart(tsNow, sPeriod, ' ') + + ' AND TestSets.enmStatus >= \'failure\'::TestStatus_T\n' + + oFilter.getWhereConditions(iOmit = TestResultFilter.kiFailReasons) + + oReportModel.getExtraSubjectWhereExpr() + + ' GROUP BY TestResultFailures.idFailureReason\n' + ' ) AS FailureReasonIDs\n' + ' LEFT OUTER JOIN FailureReasons\n' + ' ON FailureReasons.idFailureReason = FailureReasonIDs.idFailureReason\n' + ' AND FailureReasons.tsExpire = \'infinity\'::TIMESTAMP\n' + 'ORDER BY FailureReasons.idFailureReason IS NULL DESC,\n' + ' FailureReasons.sShort\n' ); + workerDoFetch(FailureReasonLogic, 'sShort', sNullDesc = 'Not given'); + + # Error counts. + oCrit = oFilter.aCriteria[TestResultFilter.kiErrorCounts]; + self._oDb.execute('SELECT TestResults.cErrors, CAST(TestResults.cErrors AS TEXT), COUNT(TestResults.idTestResult)\n' + 'FROM ( SELECT TestSets.idTestResult AS idTestResult\n' + ' FROM TestSets\n' + + oFilter.getTableJoins(iOmit = TestResultFilter.kiFailReasons) + + ''.join(' , %s\n' % (sTable,) for sTable in oReportModel.getExtraSubjectTables()) + + ' WHERE ' + self._getTimePeriodQueryPart(tsNow, sPeriod, ' ') + + oFilter.getWhereConditions(iOmit = TestResultFilter.kiFailReasons) + + oReportModel.getExtraSubjectWhereExpr() + + ' ) AS TestSetIDs\n' + ' INNER JOIN TestResults\n' + ' ON TestResults.idTestResult = TestSetIDs.idTestResult\n' + 'GROUP BY TestResults.cErrors\n' + 'ORDER BY TestResults.cErrors\n'); + + workerDoFetch(None, fIdIsName = True); + + return oFilter; + + + # + # Details view and interface. + # + + def fetchResultTree(self, idTestSet, cMaxDepth = None): + """ + Fetches the result tree for the given test set. + + Returns a tree of TestResultDataEx nodes. + Raises exception on invalid input and database issues. + """ + # Depth first, i.e. just like the XML added them. + ## @todo this still isn't performing extremely well, consider optimizations. + sQuery = self._oDb.formatBindArgs( + 'SELECT TestResults.*,\n' + ' TestResultStrTab.sValue,\n' + ' EXISTS ( SELECT idTestResultValue\n' + ' FROM TestResultValues\n' + ' WHERE TestResultValues.idTestResult = TestResults.idTestResult ) AS fHasValues,\n' + ' EXISTS ( SELECT idTestResultMsg\n' + ' FROM TestResultMsgs\n' + ' WHERE TestResultMsgs.idTestResult = TestResults.idTestResult ) AS fHasMsgs,\n' + ' EXISTS ( SELECT idTestResultFile\n' + ' FROM TestResultFiles\n' + ' WHERE TestResultFiles.idTestResult = TestResults.idTestResult ) AS fHasFiles,\n' + ' EXISTS ( SELECT idTestResult\n' + ' FROM TestResultFailures\n' + ' WHERE TestResultFailures.idTestResult = TestResults.idTestResult ) AS fHasReasons\n' + 'FROM TestResults, TestResultStrTab\n' + 'WHERE TestResults.idTestSet = %s\n' + ' AND TestResults.idStrName = TestResultStrTab.idStr\n' + , ( idTestSet, )); + if cMaxDepth is not None: + sQuery += self._oDb.formatBindArgs(' AND TestResults.iNestingDepth <= %s\n', (cMaxDepth,)); + sQuery += 'ORDER BY idTestResult ASC\n' + + self._oDb.execute(sQuery); + cRows = self._oDb.getRowCount(); + if cRows > 65536: + raise TMTooManyRows('Too many rows returned for idTestSet=%d: %d' % (idTestSet, cRows,)); + + aaoRows = self._oDb.fetchAll(); + if not aaoRows: + raise TMRowNotFound('No test results for idTestSet=%d.' % (idTestSet,)); + + # Set up the root node first. + aoRow = aaoRows[0]; + oRoot = TestResultDataEx().initFromDbRow(aoRow); + if oRoot.idTestResultParent is not None: + raise self._oDb.integrityException('The root TestResult (#%s) has a parent (#%s)!' + % (oRoot.idTestResult, oRoot.idTestResultParent)); + self._fetchResultTreeNodeExtras(oRoot, aoRow[-4], aoRow[-3], aoRow[-2], aoRow[-1]); + + # The children (if any). + dLookup = { oRoot.idTestResult: oRoot }; + oParent = oRoot; + for iRow in range(1, len(aaoRows)): + aoRow = aaoRows[iRow]; + oCur = TestResultDataEx().initFromDbRow(aoRow); + self._fetchResultTreeNodeExtras(oCur, aoRow[-4], aoRow[-3], aoRow[-2], aoRow[-1]); + + # Figure out and vet the parent. + if oParent.idTestResult != oCur.idTestResultParent: + oParent = dLookup.get(oCur.idTestResultParent, None); + if oParent is None: + raise self._oDb.integrityException('TestResult #%d is orphaned from its parent #%s.' + % (oCur.idTestResult, oCur.idTestResultParent,)); + if oParent.iNestingDepth + 1 != oCur.iNestingDepth: + raise self._oDb.integrityException('TestResult #%d has incorrect nesting depth (%d instead of %d)' + % (oCur.idTestResult, oCur.iNestingDepth, oParent.iNestingDepth + 1,)); + + # Link it up. + oCur.oParent = oParent; + oParent.aoChildren.append(oCur); + dLookup[oCur.idTestResult] = oCur; + + return (oRoot, dLookup); + + def _fetchResultTreeNodeExtras(self, oCurNode, fHasValues, fHasMsgs, fHasFiles, fHasReasons): + """ + fetchResultTree worker that fetches values, message and files for the + specified node. + """ + assert(oCurNode.aoValues == []); + assert(oCurNode.aoMsgs == []); + assert(oCurNode.aoFiles == []); + assert(oCurNode.oReason is None); + + if fHasValues: + self._oDb.execute('SELECT TestResultValues.*,\n' + ' TestResultStrTab.sValue\n' + 'FROM TestResultValues, TestResultStrTab\n' + 'WHERE TestResultValues.idTestResult = %s\n' + ' AND TestResultValues.idStrName = TestResultStrTab.idStr\n' + 'ORDER BY idTestResultValue ASC\n' + , ( oCurNode.idTestResult, )); + for aoRow in self._oDb.fetchAll(): + oCurNode.aoValues.append(TestResultValueDataEx().initFromDbRow(aoRow)); + + if fHasMsgs: + self._oDb.execute('SELECT TestResultMsgs.*,\n' + ' TestResultStrTab.sValue\n' + 'FROM TestResultMsgs, TestResultStrTab\n' + 'WHERE TestResultMsgs.idTestResult = %s\n' + ' AND TestResultMsgs.idStrMsg = TestResultStrTab.idStr\n' + 'ORDER BY idTestResultMsg ASC\n' + , ( oCurNode.idTestResult, )); + for aoRow in self._oDb.fetchAll(): + oCurNode.aoMsgs.append(TestResultMsgDataEx().initFromDbRow(aoRow)); + + if fHasFiles: + self._oDb.execute('SELECT TestResultFiles.*,\n' + ' StrTabFile.sValue AS sFile,\n' + ' StrTabDesc.sValue AS sDescription,\n' + ' StrTabKind.sValue AS sKind,\n' + ' StrTabMime.sValue AS sMime\n' + 'FROM TestResultFiles,\n' + ' TestResultStrTab AS StrTabFile,\n' + ' TestResultStrTab AS StrTabDesc,\n' + ' TestResultStrTab AS StrTabKind,\n' + ' TestResultStrTab AS StrTabMime\n' + 'WHERE TestResultFiles.idTestResult = %s\n' + ' AND TestResultFiles.idStrFile = StrTabFile.idStr\n' + ' AND TestResultFiles.idStrDescription = StrTabDesc.idStr\n' + ' AND TestResultFiles.idStrKind = StrTabKind.idStr\n' + ' AND TestResultFiles.idStrMime = StrTabMime.idStr\n' + 'ORDER BY idTestResultFile ASC\n' + , ( oCurNode.idTestResult, )); + for aoRow in self._oDb.fetchAll(): + oCurNode.aoFiles.append(TestResultFileDataEx().initFromDbRow(aoRow)); + + if fHasReasons: + if self.oFailureReasonLogic is None: + self.oFailureReasonLogic = FailureReasonLogic(self._oDb); + if self.oUserAccountLogic is None: + self.oUserAccountLogic = UserAccountLogic(self._oDb); + self._oDb.execute('SELECT *\n' + 'FROM TestResultFailures\n' + 'WHERE idTestResult = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , ( oCurNode.idTestResult, )); + if self._oDb.getRowCount() > 0: + oCurNode.oReason = TestResultFailureDataEx().initFromDbRowEx(self._oDb.fetchOne(), self.oFailureReasonLogic, + self.oUserAccountLogic); + + return True; + + + + # + # TestBoxController interface(s). + # + + def _inhumeTestResults(self, aoStack, idTestSet, sError): + """ + The test produces too much output, kill and bury it. + + Note! We leave the test set open, only the test result records are + completed. Thus, _getResultStack will return an empty stack and + cause XML processing to fail immediately, while we can still + record when it actually completed in the test set the normal way. + """ + self._oDb.dprint('** _inhumeTestResults: idTestSet=%d\n%s' % (idTestSet, self._stringifyStack(aoStack),)); + + # + # First add a message. + # + self._newFailureDetails(aoStack[0].idTestResult, idTestSet, sError, None); + + # + # The complete all open test results. + # + for oTestResult in aoStack: + oTestResult.cErrors += 1; + self._completeTestResults(oTestResult, None, TestResultData.ksTestStatus_Failure, oTestResult.cErrors); + + # A bit of paranoia. + self._oDb.execute('UPDATE TestResults\n' + 'SET cErrors = cErrors + 1,\n' + ' enmStatus = \'failure\'::TestStatus_T,\n' + ' tsElapsed = CURRENT_TIMESTAMP - tsCreated\n' + 'WHERE idTestSet = %s\n' + ' AND enmStatus = \'running\'::TestStatus_T\n' + , ( idTestSet, )); + self._oDb.commit(); + + return None; + + def strTabString(self, sString, fCommit = False): + """ + Gets the string table id for the given string, adding it if new. + + Note! A copy of this code is also in TestSetLogic. + """ + ## @todo move this and make a stored procedure for it. + self._oDb.execute('SELECT idStr\n' + 'FROM TestResultStrTab\n' + 'WHERE sValue = %s' + , (sString,)); + if self._oDb.getRowCount() == 0: + self._oDb.execute('INSERT INTO TestResultStrTab (sValue)\n' + 'VALUES (%s)\n' + 'RETURNING idStr\n' + , (sString,)); + if fCommit: + self._oDb.commit(); + return self._oDb.fetchOne()[0]; + + @staticmethod + def _stringifyStack(aoStack): + """Returns a string rep of the stack.""" + sRet = ''; + for i, _ in enumerate(aoStack): + sRet += 'aoStack[%d]=%s\n' % (i, aoStack[i]); + return sRet; + + def _getResultStack(self, idTestSet): + """ + Gets the current stack of result sets. + """ + self._oDb.execute('SELECT *\n' + 'FROM TestResults\n' + 'WHERE idTestSet = %s\n' + ' AND enmStatus = \'running\'::TestStatus_T\n' + 'ORDER BY idTestResult DESC' + , ( idTestSet, )); + aoStack = []; + for aoRow in self._oDb.fetchAll(): + aoStack.append(TestResultData().initFromDbRow(aoRow)); + + for i, _ in enumerate(aoStack): + assert aoStack[i].iNestingDepth == len(aoStack) - i - 1, self._stringifyStack(aoStack); + + return aoStack; + + def _newTestResult(self, idTestResultParent, idTestSet, iNestingDepth, tsCreated, sName, dCounts, fCommit = False): + """ + Creates a new test result. + Returns the TestResultData object for the new record. + May raise exception on database error. + """ + assert idTestResultParent is not None; + assert idTestResultParent > 1; + + # + # This isn't necessarily very efficient, but it's necessary to prevent + # a wild test or testbox from filling up the database. + # + sCountName = 'cTestResults'; + if sCountName not in dCounts: + self._oDb.execute('SELECT COUNT(idTestResult)\n' + 'FROM TestResults\n' + 'WHERE idTestSet = %s\n' + , ( idTestSet,)); + dCounts[sCountName] = self._oDb.fetchOne()[0]; + dCounts[sCountName] += 1; + if dCounts[sCountName] > config.g_kcMaxTestResultsPerTS: + raise TestResultHangingOffence('Too many sub-tests in total!'); + + sCountName = 'cTestResultsIn%d' % (idTestResultParent,); + if sCountName not in dCounts: + self._oDb.execute('SELECT COUNT(idTestResult)\n' + 'FROM TestResults\n' + 'WHERE idTestResultParent = %s\n' + , ( idTestResultParent,)); + dCounts[sCountName] = self._oDb.fetchOne()[0]; + dCounts[sCountName] += 1; + if dCounts[sCountName] > config.g_kcMaxTestResultsPerTR: + raise TestResultHangingOffence('Too many immediate sub-tests!'); + + # This is also a hanging offence. + if iNestingDepth > config.g_kcMaxTestResultDepth: + raise TestResultHangingOffence('To deep sub-test nesting!'); + + # Ditto. + if len(sName) > config.g_kcchMaxTestResultName: + raise TestResultHangingOffence('Test name is too long: %d chars - "%s"' % (len(sName), sName)); + + # + # Within bounds, do the job. + # + idStrName = self.strTabString(sName, fCommit); + self._oDb.execute('INSERT INTO TestResults (\n' + ' idTestResultParent,\n' + ' idTestSet,\n' + ' tsCreated,\n' + ' idStrName,\n' + ' iNestingDepth )\n' + 'VALUES (%s, %s, TIMESTAMP WITH TIME ZONE %s, %s, %s)\n' + 'RETURNING *\n' + , ( idTestResultParent, idTestSet, tsCreated, idStrName, iNestingDepth) ) + oData = TestResultData().initFromDbRow(self._oDb.fetchOne()); + + self._oDb.maybeCommit(fCommit); + return oData; + + def _newTestValue(self, idTestResult, idTestSet, sName, lValue, sUnit, dCounts, tsCreated = None, fCommit = False): + """ + Creates a test value. + May raise exception on database error. + """ + + # + # Bounds checking. + # + sCountName = 'cTestValues'; + if sCountName not in dCounts: + self._oDb.execute('SELECT COUNT(idTestResultValue)\n' + 'FROM TestResultValues, TestResults\n' + 'WHERE TestResultValues.idTestResult = TestResults.idTestResult\n' + ' AND TestResults.idTestSet = %s\n' + , ( idTestSet,)); + dCounts[sCountName] = self._oDb.fetchOne()[0]; + dCounts[sCountName] += 1; + if dCounts[sCountName] > config.g_kcMaxTestValuesPerTS: + raise TestResultHangingOffence('Too many values in total!'); + + sCountName = 'cTestValuesIn%d' % (idTestResult,); + if sCountName not in dCounts: + self._oDb.execute('SELECT COUNT(idTestResultValue)\n' + 'FROM TestResultValues\n' + 'WHERE idTestResult = %s\n' + , ( idTestResult,)); + dCounts[sCountName] = self._oDb.fetchOne()[0]; + dCounts[sCountName] += 1; + if dCounts[sCountName] > config.g_kcMaxTestValuesPerTR: + raise TestResultHangingOffence('Too many immediate values for one test result!'); + + if len(sName) > config.g_kcchMaxTestValueName: + raise TestResultHangingOffence('Value name is too long: %d chars - "%s"' % (len(sName), sName)); + + # + # Do the job. + # + iUnit = constants.valueunit.g_kdNameToConst.get(sUnit, constants.valueunit.NONE); + + idStrName = self.strTabString(sName, fCommit); + if tsCreated is None: + self._oDb.execute('INSERT INTO TestResultValues (\n' + ' idTestResult,\n' + ' idTestSet,\n' + ' idStrName,\n' + ' lValue,\n' + ' iUnit)\n' + 'VALUES ( %s, %s, %s, %s, %s )\n' + , ( idTestResult, idTestSet, idStrName, lValue, iUnit,) ); + else: + self._oDb.execute('INSERT INTO TestResultValues (\n' + ' idTestResult,\n' + ' idTestSet,\n' + ' tsCreated,\n' + ' idStrName,\n' + ' lValue,\n' + ' iUnit)\n' + 'VALUES ( %s, %s, TIMESTAMP WITH TIME ZONE %s, %s, %s, %s )\n' + , ( idTestResult, idTestSet, tsCreated, idStrName, lValue, iUnit,) ); + self._oDb.maybeCommit(fCommit); + return True; + + def _newFailureDetails(self, idTestResult, idTestSet, sText, dCounts, tsCreated = None, fCommit = False): + """ + Creates a record detailing cause of failure. + May raise exception on database error. + """ + + # + # Overflow protection. + # + if dCounts is not None: + sCountName = 'cTestMsgsIn%d' % (idTestResult,); + if sCountName not in dCounts: + self._oDb.execute('SELECT COUNT(idTestResultMsg)\n' + 'FROM TestResultMsgs\n' + 'WHERE idTestResult = %s\n' + , ( idTestResult,)); + dCounts[sCountName] = self._oDb.fetchOne()[0]; + dCounts[sCountName] += 1; + if dCounts[sCountName] > config.g_kcMaxTestMsgsPerTR: + raise TestResultHangingOffence('Too many messages under for one test result!'); + + if len(sText) > config.g_kcchMaxTestMsg: + raise TestResultHangingOffence('Failure details message is too long: %d chars - "%s"' % (len(sText), sText)); + + # + # Do the job. + # + idStrMsg = self.strTabString(sText, fCommit); + if tsCreated is None: + self._oDb.execute('INSERT INTO TestResultMsgs (\n' + ' idTestResult,\n' + ' idTestSet,\n' + ' idStrMsg,\n' + ' enmLevel)\n' + 'VALUES ( %s, %s, %s, %s)\n' + , ( idTestResult, idTestSet, idStrMsg, 'failure',) ); + else: + self._oDb.execute('INSERT INTO TestResultMsgs (\n' + ' idTestResult,\n' + ' idTestSet,\n' + ' tsCreated,\n' + ' idStrMsg,\n' + ' enmLevel)\n' + 'VALUES ( %s, %s, TIMESTAMP WITH TIME ZONE %s, %s, %s)\n' + , ( idTestResult, idTestSet, tsCreated, idStrMsg, 'failure',) ); + + self._oDb.maybeCommit(fCommit); + return True; + + + def _completeTestResults(self, oTestResult, tsDone, enmStatus, cErrors = 0, fCommit = False): + """ + Completes a test result. Updates the oTestResult object. + May raise exception on database error. + """ + self._oDb.dprint('** _completeTestResults: cErrors=%s tsDone=%s enmStatus=%s oTestResults=\n%s' + % (cErrors, tsDone, enmStatus, oTestResult,)); + + # + # Sanity check: No open sub tests (aoStack should make sure about this!). + # + self._oDb.execute('SELECT COUNT(idTestResult)\n' + 'FROM TestResults\n' + 'WHERE idTestResultParent = %s\n' + ' AND enmStatus = %s\n' + , ( oTestResult.idTestResult, TestResultData.ksTestStatus_Running,)); + cOpenSubTest = self._oDb.fetchOne()[0]; + assert cOpenSubTest == 0, 'cOpenSubTest=%d - %s' % (cOpenSubTest, oTestResult,); + assert oTestResult.enmStatus == TestResultData.ksTestStatus_Running; + + # + # Make sure the reporter isn't lying about successes or error counts. + # + self._oDb.execute('SELECT COALESCE(SUM(cErrors), 0)\n' + 'FROM TestResults\n' + 'WHERE idTestResultParent = %s\n' + , ( oTestResult.idTestResult, )); + cMinErrors = self._oDb.fetchOne()[0] + oTestResult.cErrors; + cErrors = max(cErrors, cMinErrors); + if cErrors > 0 and enmStatus == TestResultData.ksTestStatus_Success: + enmStatus = TestResultData.ksTestStatus_Failure + + # + # Do the update. + # + if tsDone is None: + self._oDb.execute('UPDATE TestResults\n' + 'SET cErrors = %s,\n' + ' enmStatus = %s,\n' + ' tsElapsed = CURRENT_TIMESTAMP - tsCreated\n' + 'WHERE idTestResult = %s\n' + 'RETURNING tsElapsed' + , ( cErrors, enmStatus, oTestResult.idTestResult,) ); + else: + self._oDb.execute('UPDATE TestResults\n' + 'SET cErrors = %s,\n' + ' enmStatus = %s,\n' + ' tsElapsed = TIMESTAMP WITH TIME ZONE %s - tsCreated\n' + 'WHERE idTestResult = %s\n' + 'RETURNING tsElapsed' + , ( cErrors, enmStatus, tsDone, oTestResult.idTestResult,) ); + + oTestResult.tsElapsed = self._oDb.fetchOne()[0]; + oTestResult.enmStatus = enmStatus; + oTestResult.cErrors = cErrors; + + self._oDb.maybeCommit(fCommit); + return None; + + def _doPopHint(self, aoStack, cStackEntries, dCounts, idTestSet): + """ Executes a PopHint. """ + assert cStackEntries >= 0; + while len(aoStack) > cStackEntries: + if aoStack[0].enmStatus == TestResultData.ksTestStatus_Running: + self._newFailureDetails(aoStack[0].idTestResult, idTestSet, 'XML error: Missing </Test>', dCounts); + self._completeTestResults(aoStack[0], tsDone = None, cErrors = 1, + enmStatus = TestResultData.ksTestStatus_Failure, fCommit = True); + aoStack.pop(0); + return True; + + + @staticmethod + def _validateElement(sName, dAttribs, fClosed): + """ + Validates an element and its attributes. + """ + + # + # Validate attributes by name. + # + + # Validate integer attributes. + for sAttr in [ 'errors', 'testdepth' ]: + if sAttr in dAttribs: + try: + _ = int(dAttribs[sAttr]); + except: + return 'Element %s has an invalid %s attribute value: %s.' % (sName, sAttr, dAttribs[sAttr],); + + # Validate long attributes. + for sAttr in [ 'value', ]: + if sAttr in dAttribs: + try: + _ = long(dAttribs[sAttr]); # pylint: disable=redefined-variable-type + except: + return 'Element %s has an invalid %s attribute value: %s.' % (sName, sAttr, dAttribs[sAttr],); + + # Validate string attributes. + for sAttr in [ 'name', 'text' ]: # 'unit' can be zero length. + if sAttr in dAttribs and not dAttribs[sAttr]: + return 'Element %s has an empty %s attribute value.' % (sName, sAttr,); + + # Validate the timestamp attribute. + if 'timestamp' in dAttribs: + (dAttribs['timestamp'], sError) = ModelDataBase.validateTs(dAttribs['timestamp'], fAllowNull = False); + if sError is not None: + return 'Element %s has an invalid timestamp ("%s"): %s' % (sName, dAttribs['timestamp'], sError,); + + + # + # Check that attributes that are required are present. + # We ignore extra attributes. + # + dElementAttribs = \ + { + 'Test': [ 'timestamp', 'name', ], + 'Value': [ 'timestamp', 'name', 'unit', 'value', ], + 'FailureDetails': [ 'timestamp', 'text', ], + 'Passed': [ 'timestamp', ], + 'Skipped': [ 'timestamp', ], + 'Failed': [ 'timestamp', 'errors', ], + 'TimedOut': [ 'timestamp', 'errors', ], + 'End': [ 'timestamp', ], + 'PushHint': [ 'testdepth', ], + 'PopHint': [ 'testdepth', ], + }; + if sName not in dElementAttribs: + return 'Unknown element "%s".' % (sName,); + for sAttr in dElementAttribs[sName]: + if sAttr not in dAttribs: + return 'Element %s requires attribute "%s".' % (sName, sAttr); + + # + # Only the Test element can (and must) remain open. + # + if sName == 'Test' and fClosed: + return '<Test/> is not allowed.'; + if sName != 'Test' and not fClosed: + return 'All elements except <Test> must be closed.'; + + return None; + + @staticmethod + def _parseElement(sElement): + """ + Parses an element. + + """ + # + # Element level bits. + # + sName = sElement.split()[0]; + sElement = sElement[len(sName):]; + + fClosed = sElement[-1] == '/'; + if fClosed: + sElement = sElement[:-1]; + + # + # Attributes. + # + sError = None; + dAttribs = {}; + sElement = sElement.strip(); + while sElement: + # Extract attribute name. + off = sElement.find('='); + if off < 0 or not sElement[:off].isalnum(): + sError = 'Attributes shall have alpha numberical names and have values.'; + break; + sAttr = sElement[:off]; + + # Extract attribute value. + if off + 2 >= len(sElement) or sElement[off + 1] != '"': + sError = 'Attribute (%s) value is missing or not in double quotes.' % (sAttr,); + break; + off += 2; + offEndQuote = sElement.find('"', off); + if offEndQuote < 0: + sError = 'Attribute (%s) value is missing end quotation mark.' % (sAttr,); + break; + sValue = sElement[off:offEndQuote]; + + # Check for duplicates. + if sAttr in dAttribs: + sError = 'Attribute "%s" appears more than once.' % (sAttr,); + break; + + # Unescape the value. + sValue = sValue.replace('<', '<'); + sValue = sValue.replace('>', '>'); + sValue = sValue.replace(''', '\''); + sValue = sValue.replace('"', '"'); + sValue = sValue.replace('
', '\n'); + sValue = sValue.replace('
', '\r'); + sValue = sValue.replace('&', '&'); # last + + # Done. + dAttribs[sAttr] = sValue; + + # advance + sElement = sElement[offEndQuote + 1:]; + sElement = sElement.lstrip(); + + # + # Validate the element before we return. + # + if sError is None: + sError = TestResultLogic._validateElement(sName, dAttribs, fClosed); + + return (sName, dAttribs, sError) + + def _handleElement(self, sName, dAttribs, idTestSet, aoStack, aaiHints, dCounts): + """ + Worker for processXmlStream that handles one element. + + Returns None on success, error string on bad XML or similar. + Raises exception on hanging offence and on database error. + """ + if sName == 'Test': + iNestingDepth = aoStack[0].iNestingDepth + 1 if aoStack else 0; + aoStack.insert(0, self._newTestResult(idTestResultParent = aoStack[0].idTestResult, idTestSet = idTestSet, + tsCreated = dAttribs['timestamp'], sName = dAttribs['name'], + iNestingDepth = iNestingDepth, dCounts = dCounts, fCommit = True) ); + + elif sName == 'Value': + self._newTestValue(idTestResult = aoStack[0].idTestResult, idTestSet = idTestSet, tsCreated = dAttribs['timestamp'], + sName = dAttribs['name'], sUnit = dAttribs['unit'], lValue = long(dAttribs['value']), + dCounts = dCounts, fCommit = True); + + elif sName == 'FailureDetails': + self._newFailureDetails(idTestResult = aoStack[0].idTestResult, idTestSet = idTestSet, + tsCreated = dAttribs['timestamp'], sText = dAttribs['text'], dCounts = dCounts, + fCommit = True); + + elif sName == 'Passed': + self._completeTestResults(aoStack[0], tsDone = dAttribs['timestamp'], + enmStatus = TestResultData.ksTestStatus_Success, fCommit = True); + + elif sName == 'Skipped': + self._completeTestResults(aoStack[0], tsDone = dAttribs['timestamp'], + enmStatus = TestResultData.ksTestStatus_Skipped, fCommit = True); + + elif sName == 'Failed': + self._completeTestResults(aoStack[0], tsDone = dAttribs['timestamp'], cErrors = int(dAttribs['errors']), + enmStatus = TestResultData.ksTestStatus_Failure, fCommit = True); + + elif sName == 'TimedOut': + self._completeTestResults(aoStack[0], tsDone = dAttribs['timestamp'], cErrors = int(dAttribs['errors']), + enmStatus = TestResultData.ksTestStatus_TimedOut, fCommit = True); + + elif sName == 'End': + self._completeTestResults(aoStack[0], tsDone = dAttribs['timestamp'], + cErrors = int(dAttribs.get('errors', '1')), + enmStatus = TestResultData.ksTestStatus_Success, fCommit = True); + + elif sName == 'PushHint': + if len(aaiHints) > 1: + return 'PushHint cannot be nested.' + + aaiHints.insert(0, [len(aoStack), int(dAttribs['testdepth'])]); + + elif sName == 'PopHint': + if not aaiHints: + return 'No hint to pop.' + + iDesiredTestDepth = int(dAttribs['testdepth']); + cStackEntries, iTestDepth = aaiHints.pop(0); + self._doPopHint(aoStack, cStackEntries, dCounts, idTestSet); # Fake the necessary '<End/></Test>' tags. + if iDesiredTestDepth != iTestDepth: + return 'PopHint tag has different testdepth: %d, on stack %d.' % (iDesiredTestDepth, iTestDepth); + else: + return 'Unexpected element "%s".' % (sName,); + return None; + + + def processXmlStream(self, sXml, idTestSet): + """ + Processes the "XML" stream section given in sXml. + + The sXml isn't a complete XML document, even should we save up all sXml + for a given set, they may not form a complete and well formed XML + document since the test may be aborted, abend or simply be buggy. We + therefore do our own parsing and treat the XML tags as commands more + than anything else. + + Returns (sError, fUnforgivable), where sError is None on success. + May raise database exception. + """ + aoStack = self._getResultStack(idTestSet); # [0] == top; [-1] == bottom. + if not aoStack: + return ('No open results', True); + self._oDb.dprint('** processXmlStream len(aoStack)=%s' % (len(aoStack),)); + #self._oDb.dprint('processXmlStream: %s' % (self._stringifyStack(aoStack),)); + #self._oDb.dprint('processXmlStream: sXml=%s' % (sXml,)); + + dCounts = {}; + aaiHints = []; + sError = None; + + fExpectCloseTest = False; + sXml = sXml.strip(); + while sXml: + if sXml.startswith('</Test>'): # Only closing tag. + offNext = len('</Test>'); + if len(aoStack) <= 1: + sError = 'Trying to close the top test results.' + break; + # ASSUMES that we've just seen an <End/>, <Passed/>, <Failed/>, + # <TimedOut/> or <Skipped/> tag earlier in this call! + if aoStack[0].enmStatus == TestResultData.ksTestStatus_Running or not fExpectCloseTest: + sError = 'Missing <End/>, <Passed/>, <Failed/>, <TimedOut/> or <Skipped/> tag.'; + break; + aoStack.pop(0); + fExpectCloseTest = False; + + elif fExpectCloseTest: + sError = 'Expected </Test>.' + break; + + elif sXml.startswith('<?xml '): # Ignore (included files). + offNext = sXml.find('?>'); + if offNext < 0: + sError = 'Unterminated <?xml ?> element.'; + break; + offNext += 2; + + elif sXml[0] == '<': + # Parse and check the tag. + if not sXml[1].isalpha(): + sError = 'Malformed element.'; + break; + offNext = sXml.find('>') + if offNext < 0: + sError = 'Unterminated element.'; + break; + (sName, dAttribs, sError) = self._parseElement(sXml[1:offNext]); + offNext += 1; + if sError is not None: + break; + + # Handle it. + try: + sError = self._handleElement(sName, dAttribs, idTestSet, aoStack, aaiHints, dCounts); + except TestResultHangingOffence as oXcpt: + self._inhumeTestResults(aoStack, idTestSet, str(oXcpt)); + return (str(oXcpt), True); + + + fExpectCloseTest = sName in [ 'End', 'Passed', 'Failed', 'TimedOut', 'Skipped', ]; + else: + sError = 'Unexpected content.'; + break; + + # Advance. + sXml = sXml[offNext:]; + sXml = sXml.lstrip(); + + # + # Post processing checks. + # + if sError is None and fExpectCloseTest: + sError = 'Expected </Test> before the end of the XML section.' + elif sError is None and aaiHints: + sError = 'Expected </PopHint> before the end of the XML section.' + if aaiHints: + self._doPopHint(aoStack, aaiHints[-1][0], dCounts, idTestSet); + + # + # Log the error. + # + if sError is not None: + SystemLogLogic(self._oDb).addEntry(SystemLogData.ksEvent_XmlResultMalformed, + 'idTestSet=%s idTestResult=%s XML="%s" %s' + % ( idTestSet, + aoStack[0].idTestResult if aoStack else -1, + sXml[:min(len(sXml), 30)], + sError, ), + cHoursRepeat = 6, fCommit = True); + return (sError, False); + + + + + +# +# Unit testing. +# + +# pylint: disable=missing-docstring +class TestResultDataTestCase(ModelDataBaseTestCase): + def setUp(self): + self.aoSamples = [TestResultData(),]; + +class TestResultValueDataTestCase(ModelDataBaseTestCase): + def setUp(self): + self.aoSamples = [TestResultValueData(),]; + +if __name__ == '__main__': + unittest.main(); + # not reached. + diff --git a/src/VBox/ValidationKit/testmanager/core/testset.py b/src/VBox/ValidationKit/testmanager/core/testset.py new file mode 100755 index 00000000..e9e408fe --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/testset.py @@ -0,0 +1,869 @@ +# -*- coding: utf-8 -*- +# $Id: testset.py $ + +""" +Test Manager - TestSet. +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Standard python imports. +import os; +import zipfile; +import unittest; + +# Validation Kit imports. +from common import utils; +from testmanager import config; +from testmanager.core import db; +from testmanager.core.base import ModelDataBase, ModelDataBaseTestCase, ModelLogicBase, \ + TMExceptionBase, TMTooManyRows, TMRowNotFound; +from testmanager.core.testbox import TestBoxData; +from testmanager.core.testresults import TestResultFileDataEx; + + +class TestSetData(ModelDataBase): + """ + TestSet Data. + """ + + ## @name TestStatus_T + # @{ + ksTestStatus_Running = 'running'; + ksTestStatus_Success = 'success'; + ksTestStatus_Skipped = 'skipped'; + ksTestStatus_BadTestBox = 'bad-testbox'; + ksTestStatus_Aborted = 'aborted'; + ksTestStatus_Failure = 'failure'; + ksTestStatus_TimedOut = 'timed-out'; + ksTestStatus_Rebooted = 'rebooted'; + ## @} + + ## List of relatively harmless (to testgroup/case) statuses. + kasHarmlessTestStatuses = [ ksTestStatus_Skipped, ksTestStatus_BadTestBox, ksTestStatus_Aborted, ]; + ## List of bad statuses. + kasBadTestStatuses = [ ksTestStatus_Failure, ksTestStatus_TimedOut, ksTestStatus_Rebooted, ]; + + ksIdAttr = 'idTestSet'; + + ksParam_idTestSet = 'TestSet_idTestSet'; + ksParam_tsConfig = 'TestSet_tsConfig'; + ksParam_tsCreated = 'TestSet_tsCreated'; + ksParam_tsDone = 'TestSet_tsDone'; + ksParam_enmStatus = 'TestSet_enmStatus'; + ksParam_idBuild = 'TestSet_idBuild'; + ksParam_idBuildCategory = 'TestSet_idBuildCategory'; + ksParam_idBuildTestSuite = 'TestSet_idBuildTestSuite'; + ksParam_idGenTestBox = 'TestSet_idGenTestBox'; + ksParam_idTestBox = 'TestSet_idTestBox'; + ksParam_idSchedGroup = 'TestSet_idSchedGroup'; + ksParam_idTestGroup = 'TestSet_idTestGroup'; + ksParam_idGenTestCase = 'TestSet_idGenTestCase'; + ksParam_idTestCase = 'TestSet_idTestCase'; + ksParam_idGenTestCaseArgs = 'TestSet_idGenTestCaseArgs'; + ksParam_idTestCaseArgs = 'TestSet_idTestCaseArgs'; + ksParam_idTestResult = 'TestSet_idTestResult'; + ksParam_sBaseFilename = 'TestSet_sBaseFilename'; + ksParam_iGangMemberNo = 'TestSet_iGangMemberNo'; + ksParam_idTestSetGangLeader = 'TestSet_idTestSetGangLeader'; + + kasAllowNullAttributes = [ 'tsDone', 'idBuildTestSuite', 'idTestSetGangLeader' ]; + kasValidValues_enmStatus = [ + ksTestStatus_Running, + ksTestStatus_Success, + ksTestStatus_Skipped, + ksTestStatus_BadTestBox, + ksTestStatus_Aborted, + ksTestStatus_Failure, + ksTestStatus_TimedOut, + ksTestStatus_Rebooted, + ]; + kiMin_iGangMemberNo = 0; + kiMax_iGangMemberNo = 1023; + + + kcDbColumns = 20; + + def __init__(self): + ModelDataBase.__init__(self); + + # + # Initialize with defaults. + # See the database for explanations of each of these fields. + # + self.idTestSet = None; + self.tsConfig = None; + self.tsCreated = None; + self.tsDone = None; + self.enmStatus = 'running'; + self.idBuild = None; + self.idBuildCategory = None; + self.idBuildTestSuite = None; + self.idGenTestBox = None; + self.idTestBox = None; + self.idSchedGroup = None; + self.idTestGroup = None; + self.idGenTestCase = None; + self.idTestCase = None; + self.idGenTestCaseArgs = None; + self.idTestCaseArgs = None; + self.idTestResult = None; + self.sBaseFilename = None; + self.iGangMemberNo = 0; + self.idTestSetGangLeader = None; + + def initFromDbRow(self, aoRow): + """ + Internal worker for initFromDbWithId and initFromDbWithGenId as well as + TestBoxSetLogic. + """ + + if aoRow is None: + raise TMRowNotFound('TestSet not found.'); + + self.idTestSet = aoRow[0]; + self.tsConfig = aoRow[1]; + self.tsCreated = aoRow[2]; + self.tsDone = aoRow[3]; + self.enmStatus = aoRow[4]; + self.idBuild = aoRow[5]; + self.idBuildCategory = aoRow[6]; + self.idBuildTestSuite = aoRow[7]; + self.idGenTestBox = aoRow[8]; + self.idTestBox = aoRow[9]; + self.idSchedGroup = aoRow[10]; + self.idTestGroup = aoRow[11]; + self.idGenTestCase = aoRow[12]; + self.idTestCase = aoRow[13]; + self.idGenTestCaseArgs = aoRow[14]; + self.idTestCaseArgs = aoRow[15]; + self.idTestResult = aoRow[16]; + self.sBaseFilename = aoRow[17]; + self.iGangMemberNo = aoRow[18]; + self.idTestSetGangLeader = aoRow[19]; + return self; + + + def initFromDbWithId(self, oDb, idTestSet): + """ + Initialize the object from the database. + """ + oDb.execute('SELECT *\n' + 'FROM TestSets\n' + 'WHERE idTestSet = %s\n' + , (idTestSet, ) ); + aoRow = oDb.fetchOne() + if aoRow is None: + raise TMRowNotFound('idTestSet=%s not found' % (idTestSet,)); + return self.initFromDbRow(aoRow); + + + def openFile(self, sFilename, sMode = 'rb'): + """ + Opens a file. + + Returns (oFile, cbFile, fIsStream) on success. + Returns (None, sErrorMsg, None) on failure. + Will not raise exceptions, unless the class instance is invalid. + """ + assert sMode in [ 'rb', 'r', 'rU' ]; + + # Try raw file first. + sFile1 = os.path.join(config.g_ksFileAreaRootDir, self.sBaseFilename + '-' + sFilename); + try: + oFile = open(sFile1, sMode); # pylint: disable=consider-using-with,unspecified-encoding + return (oFile, os.fstat(oFile.fileno()).st_size, False); + except Exception as oXcpt1: + # Try the zip archive next. + sFile2 = os.path.join(config.g_ksZipFileAreaRootDir, self.sBaseFilename + '.zip'); + try: + oZipFile = zipfile.ZipFile(sFile2, 'r'); # pylint: disable=consider-using-with + oFile = oZipFile.open(sFilename, sMode if sMode != 'rb' else 'r'); # pylint: disable=consider-using-with + cbFile = oZipFile.getinfo(sFilename).file_size; + return (oFile, cbFile, True); + except Exception as oXcpt2: + # Construct a meaningful error message. + try: + if os.path.exists(sFile1): + return (None, 'Error opening "%s": %s' % (sFile1, oXcpt1), None); + if not os.path.exists(sFile2): + return (None, 'File "%s" not found. [%s, %s]' % (sFilename, sFile1, sFile2,), None); + return (None, 'Error opening "%s" inside "%s": %s' % (sFilename, sFile2, oXcpt2), None); + except Exception as oXcpt3: + return (None, 'OMG! %s; %s; %s' % (oXcpt1, oXcpt2, oXcpt3,), None); + return (None, 'Code not reachable!', None); + + def createFile(self, sFilename, sMode = 'wb'): + """ + Creates a new file. + + Returns oFile on success. + Returns sErrorMsg on failure. + """ + assert sMode in [ 'wb', 'w', 'wU' ]; + + # Try raw file first. + sFile1 = os.path.join(config.g_ksFileAreaRootDir, self.sBaseFilename + '-' + sFilename); + try: + if not os.path.exists(os.path.dirname(sFile1)): + os.makedirs(os.path.dirname(sFile1), 0o755); + oFile = open(sFile1, sMode); # pylint: disable=consider-using-with,unspecified-encoding + except Exception as oXcpt1: + return str(oXcpt1); + return oFile; + + @staticmethod + def findLogOffsetForTimestamp(sLogContent, tsTimestamp, offStart = 0, fAfter = False): + """ + Log parsing utility function for finding the offset for the given timestamp. + + We ASSUME the log lines are prefixed with UTC timestamps on the format + '09:43:55.789353'. + + Return index into the sLogContent string, 0 if not found. + """ + # Turn tsTimestamp into a string compatible with what we expect to find in the log. + oTsZulu = db.dbTimestampToZuluDatetime(tsTimestamp); + sWantedTs = oTsZulu.strftime('%H:%M:%S.%f'); + assert len(sWantedTs) == 15; + + # Now loop thru the string, line by line. + offRet = offStart; + off = offStart; + while True: + sThisTs = sLogContent[off : off + 15]; + if len(sThisTs) >= 15 \ + and sThisTs[2] == ':' \ + and sThisTs[5] == ':' \ + and sThisTs[8] == '.' \ + and sThisTs[14] in '0123456789': + if sThisTs < sWantedTs: + offRet = off; + elif sThisTs == sWantedTs: + if not fAfter: + return off; + offRet = off; + else: + if fAfter: + offRet = off; + break; + + # next line. + off = sLogContent.find('\n', off); + if off < 0: + if fAfter: + offRet = len(sLogContent); + break; + off += 1; + + return offRet; + + @staticmethod + def extractLogSection(sLogContent, tsStart, tsLast): + """ + Returns log section from tsStart to tsLast (or all if we cannot make sense of it). + """ + offStart = TestSetData.findLogOffsetForTimestamp(sLogContent, tsStart); + offEnd = TestSetData.findLogOffsetForTimestamp(sLogContent, tsLast, offStart, fAfter = True); + return sLogContent[offStart : offEnd]; + + @staticmethod + def extractLogSectionElapsed(sLogContent, tsStart, tsElapsed): + """ + Returns log section from tsStart and tsElapsed forward (or all if we cannot make sense of it). + """ + tsStart = db.dbTimestampToZuluDatetime(tsStart); + tsLast = tsStart + tsElapsed; + return TestSetData.extractLogSection(sLogContent, tsStart, tsLast); + + + +class TestSetLogic(ModelLogicBase): + """ + TestSet logic. + """ + + + def __init__(self, oDb): + ModelLogicBase.__init__(self, oDb); + + + def tryFetch(self, idTestSet): + """ + Attempts to fetch a test set. + + Returns a TestSetData object on success. + Returns None if no status was found. + Raises exception on other errors. + """ + self._oDb.execute('SELECT *\n' + 'FROM TestSets\n' + 'WHERE idTestSet = %s\n', + (idTestSet,)); + if self._oDb.getRowCount() == 0: + return None; + oData = TestSetData(); + return oData.initFromDbRow(self._oDb.fetchOne()); + + def strTabString(self, sString, fCommit = False): + """ + Gets the string table id for the given string, adding it if new. + """ + ## @todo move this and make a stored procedure for it. + self._oDb.execute('SELECT idStr\n' + 'FROM TestResultStrTab\n' + 'WHERE sValue = %s' + , (sString,)); + if self._oDb.getRowCount() == 0: + self._oDb.execute('INSERT INTO TestResultStrTab (sValue)\n' + 'VALUES (%s)\n' + 'RETURNING idStr\n' + , (sString,)); + if fCommit: + self._oDb.commit(); + return self._oDb.fetchOne()[0]; + + def complete(self, idTestSet, sStatus, fCommit = False): + """ + Completes the testset. + Returns the test set ID of the gang leader, None if no gang involvement. + Raises exceptions on database errors and invalid input. + """ + + assert sStatus != TestSetData.ksTestStatus_Running; + + # + # Get the basic test set data and check if there is anything to do here. + # + oData = TestSetData().initFromDbWithId(self._oDb, idTestSet); + if oData.enmStatus != TestSetData.ksTestStatus_Running: + raise TMExceptionBase('TestSet %s is already completed as %s.' % (idTestSet, oData.enmStatus)); + if oData.idTestResult is None: + raise self._oDb.integrityException('idTestResult is NULL for TestSet %u' % (idTestSet,)); + + # + # Close open sub test results, count these as errors. + # Note! No need to propagate error counts here. Only one tree line will + # have open sets, and it will go all the way to the root. + # + self._oDb.execute('SELECT idTestResult\n' + 'FROM TestResults\n' + 'WHERE idTestSet = %s\n' + ' AND enmStatus = %s\n' + ' AND idTestResult <> %s\n' + 'ORDER BY idTestResult DESC\n' + , (idTestSet, TestSetData.ksTestStatus_Running, oData.idTestResult)); + aaoRows = self._oDb.fetchAll(); + if aaoRows: + idStr = self.strTabString('Unclosed test result', fCommit = fCommit); + for aoRow in aaoRows: + self._oDb.execute('UPDATE TestResults\n' + 'SET enmStatus = \'failure\',\n' + ' tsElapsed = CURRENT_TIMESTAMP - tsCreated,\n' + ' cErrors = cErrors + 1\n' + 'WHERE idTestResult = %s\n' + , (aoRow[0],)); + self._oDb.execute('INSERT INTO TestResultMsgs (idTestResult, idTestSet, idStrMsg, enmLevel)\n' + 'VALUES ( %s, %s, %s, \'failure\'::TestResultMsgLevel_T)\n' + , (aoRow[0], idTestSet, idStr,)); + + # + # If it's a success result, check it against error counters. + # + if sStatus not in TestSetData.kasBadTestStatuses: + self._oDb.execute('SELECT COUNT(*)\n' + 'FROM TestResults\n' + 'WHERE idTestSet = %s\n' + ' AND cErrors > 0\n' + , (idTestSet,)); + cErrors = self._oDb.fetchOne()[0]; + if cErrors > 0: + sStatus = TestSetData.ksTestStatus_Failure; + + # + # If it's an pure 'failure', check for timeouts and propagate it. + # + if sStatus == TestSetData.ksTestStatus_Failure: + self._oDb.execute('SELECT COUNT(*)\n' + 'FROM TestResults\n' + 'WHERE idTestSet = %s\n' + ' AND enmStatus = %s\n' + , ( idTestSet, TestSetData.ksTestStatus_TimedOut, )); + if self._oDb.fetchOne()[0] > 0: + sStatus = TestSetData.ksTestStatus_TimedOut; + + # + # Complete the top level test result and then the test set. + # + self._oDb.execute('UPDATE TestResults\n' + 'SET cErrors = (SELECT COALESCE(SUM(cErrors), 0)\n' + ' FROM TestResults\n' + ' WHERE idTestResultParent = %s)\n' + 'WHERE idTestResult = %s\n' + 'RETURNING cErrors\n' + , (oData.idTestResult, oData.idTestResult)); + cErrors = self._oDb.fetchOne()[0]; + if cErrors == 0 and sStatus in TestSetData.kasBadTestStatuses: + self._oDb.execute('UPDATE TestResults\n' + 'SET cErrors = 1\n' + 'WHERE idTestResult = %s\n' + , (oData.idTestResult,)); + elif cErrors > 0 and sStatus not in TestSetData.kasBadTestStatuses: + sStatus = TestSetData.ksTestStatus_Failure; # Impossible. + self._oDb.execute('UPDATE TestResults\n' + 'SET enmStatus = %s,\n' + ' tsElapsed = CURRENT_TIMESTAMP - tsCreated\n' + 'WHERE idTestResult = %s\n' + , (sStatus, oData.idTestResult,)); + + self._oDb.execute('UPDATE TestSets\n' + 'SET enmStatus = %s,\n' + ' tsDone = CURRENT_TIMESTAMP\n' + 'WHERE idTestSet = %s\n' + , (sStatus, idTestSet,)); + + self._oDb.maybeCommit(fCommit); + return oData.idTestSetGangLeader; + + def completeAsAbandoned(self, idTestSet, fCommit = False): + """ + Completes the testset as abandoned if necessary. + + See scenario #9: + file://../../docs/AutomaticTestingRevamp.html#cleaning-up-abandond-testcase + + Returns True if successfully completed as abandond, False if it's already + completed, and raises exceptions under exceptional circumstances. + """ + + # + # Get the basic test set data and check if there is anything to do here. + # + oData = self.tryFetch(idTestSet); + if oData is None: + return False; + if oData.enmStatus != TestSetData.ksTestStatus_Running: + return False; + + if oData.idTestResult is not None: + # + # Clean up test results, adding a message why they failed. + # + self._oDb.execute('UPDATE TestResults\n' + 'SET enmStatus = \'failure\',\n' + ' tsElapsed = CURRENT_TIMESTAMP - tsCreated,\n' + ' cErrors = cErrors + 1\n' + 'WHERE idTestSet = %s\n' + ' AND enmStatus = \'running\'::TestStatus_T\n' + , (idTestSet,)); + + idStr = self.strTabString('The test was abandond by the testbox', fCommit = fCommit); + self._oDb.execute('INSERT INTO TestResultMsgs (idTestResult, idTestSet, idStrMsg, enmLevel)\n' + 'VALUES ( %s, %s, %s, \'failure\'::TestResultMsgLevel_T)\n' + , (oData.idTestResult, idTestSet, idStr,)); + + # + # Complete the testset. + # + self._oDb.execute('UPDATE TestSets\n' + 'SET enmStatus = \'failure\',\n' + ' tsDone = CURRENT_TIMESTAMP\n' + 'WHERE idTestSet = %s\n' + ' AND enmStatus = \'running\'::TestStatus_T\n' + , (idTestSet,)); + + self._oDb.maybeCommit(fCommit); + return True; + + def completeAsGangGatheringTimeout(self, idTestSet, fCommit = False): + """ + Completes the testset with a gang-gathering timeout. + Raises exceptions on database errors and invalid input. + """ + # + # Get the basic test set data and check if there is anything to do here. + # + oData = TestSetData().initFromDbWithId(self._oDb, idTestSet); + if oData.enmStatus != TestSetData.ksTestStatus_Running: + raise TMExceptionBase('TestSet %s is already completed as %s.' % (idTestSet, oData.enmStatus)); + if oData.idTestResult is None: + raise self._oDb.integrityException('idTestResult is NULL for TestSet %u' % (idTestSet,)); + + # + # Complete the top level test result and then the test set. + # + self._oDb.execute('UPDATE TestResults\n' + 'SET enmStatus = \'failure\',\n' + ' tsElapsed = CURRENT_TIMESTAMP - tsCreated,\n' + ' cErrors = cErrors + 1\n' + 'WHERE idTestSet = %s\n' + ' AND enmStatus = \'running\'::TestStatus_T\n' + , (idTestSet,)); + + idStr = self.strTabString('Gang gathering timed out', fCommit = fCommit); + self._oDb.execute('INSERT INTO TestResultMsgs (idTestResult, idTestSet, idStrMsg, enmLevel)\n' + 'VALUES ( %s, %s, %s, \'failure\'::TestResultMsgLevel_T)\n' + , (oData.idTestResult, idTestSet, idStr,)); + + self._oDb.execute('UPDATE TestSets\n' + 'SET enmStatus = \'failure\',\n' + ' tsDone = CURRENT_TIMESTAMP\n' + 'WHERE idTestSet = %s\n' + , (idTestSet,)); + + self._oDb.maybeCommit(fCommit); + return True; + + def createFile(self, oTestSet, sName, sMime, sKind, sDesc, cbFile, fCommit = False): # pylint: disable=too-many-locals + """ + Creates a file and associating with the current test result record in + the test set. + + Returns file object that the file content can be written to. + Raises exception on database error, I/O errors, if there are too many + files in the test set or if they take up too much disk space. + + The caller (testboxdisp.py) is expected to do basic input validation, + so we skip that and get on with the bits only we can do. + """ + + # + # Furhter input and limit checks. + # + if oTestSet.enmStatus != TestSetData.ksTestStatus_Running: + raise TMExceptionBase('Cannot create files on a test set with status "%s".' % (oTestSet.enmStatus,)); + + self._oDb.execute('SELECT TestResultStrTab.sValue\n' + 'FROM TestResultFiles,\n' + ' TestResults,\n' + ' TestResultStrTab\n' + 'WHERE TestResults.idTestSet = %s\n' + ' AND TestResultFiles.idTestResult = TestResults.idTestResult\n' + ' AND TestResultStrTab.idStr = TestResultFiles.idStrFile\n' + , ( oTestSet.idTestSet,)); + if self._oDb.getRowCount() + 1 > config.g_kcMaxUploads: + raise TMExceptionBase('Uploaded too many files already (%d).' % (self._oDb.getRowCount(),)); + + dFiles = {} + cbTotalFiles = 0; + for aoRow in self._oDb.fetchAll(): + dFiles[aoRow[0].lower()] = 1; # For determining a unique filename further down. + sFile = os.path.join(config.g_ksFileAreaRootDir, oTestSet.sBaseFilename + '-' + aoRow[0]); + try: + cbTotalFiles += os.path.getsize(sFile); + except: + cbTotalFiles += config.g_kcMbMaxUploadSingle * 1048576; + if (cbTotalFiles + cbFile + 1048575) / 1048576 > config.g_kcMbMaxUploadTotal: + raise TMExceptionBase('Will exceed total upload limit: %u bytes + %u bytes > %s MiB.' \ + % (cbTotalFiles, cbFile, config.g_kcMbMaxUploadTotal)); + + # + # Create a new file. + # + self._oDb.execute('SELECT idTestResult\n' + 'FROM TestResults\n' + 'WHERE idTestSet = %s\n' + ' AND enmStatus = \'running\'::TestStatus_T\n' + 'ORDER BY idTestResult DESC\n' + 'LIMIT 1\n' + % ( oTestSet.idTestSet, )); + if self._oDb.getRowCount() < 1: + raise TMExceptionBase('No open test results - someone committed a capital offence or we ran into a race.'); + idTestResult = self._oDb.fetchOne()[0]; + + if sName.lower() in dFiles: + # Note! There is in theory a race here, but that's something the + # test driver doing parallel upload with non-unique names + # should worry about. The TD should always avoid this path. + sOrgName = sName; + for i in range(2, config.g_kcMaxUploads + 6): + sName = '%s-%s' % (i, sName,); + if sName not in dFiles: + break; + sName = None; + if sName is None: + raise TMExceptionBase('Failed to find unique name for %s.' % (sOrgName,)); + + self._oDb.execute('INSERT INTO TestResultFiles(idTestResult, idTestSet, idStrFile, idStrDescription,\n' + ' idStrKind, idStrMime)\n' + 'VALUES (%s, %s, %s, %s, %s, %s)\n' + , ( idTestResult, + oTestSet.idTestSet, + self.strTabString(sName), + self.strTabString(sDesc), + self.strTabString(sKind), + self.strTabString(sMime), + )); + + oFile = oTestSet.createFile(sName, 'wb'); + if utils.isString(oFile): + raise TMExceptionBase('Error creating "%s": %s' % (sName, oFile)); + self._oDb.maybeCommit(fCommit); + return oFile; + + def getGang(self, idTestSetGangLeader): + """ + Returns an array of TestBoxData object representing the gang for the given testset. + """ + self._oDb.execute('SELECT TestBoxesWithStrings.*\n' + 'FROM TestBoxesWithStrings,\n' + ' TestSets' + 'WHERE TestSets.idTestSetGangLeader = %s\n' + ' AND TestSets.idGenTestBox = TestBoxesWithStrings.idGenTestBox\n' + 'ORDER BY iGangMemberNo ASC\n' + , ( idTestSetGangLeader,)); + aaoRows = self._oDb.fetchAll(); + aoTestBoxes = []; + for aoRow in aaoRows: + aoTestBoxes.append(TestBoxData().initFromDbRow(aoRow)); + return aoTestBoxes; + + def getFile(self, idTestSet, idTestResultFile): + """ + Gets the TestResultFileEx corresponding to idTestResultFile. + + Raises an exception if the file wasn't found, doesn't belong to + idTestSet, and on DB error. + """ + self._oDb.execute('SELECT TestResultFiles.*,\n' + ' StrTabFile.sValue AS sFile,\n' + ' StrTabDesc.sValue AS sDescription,\n' + ' StrTabKind.sValue AS sKind,\n' + ' StrTabMime.sValue AS sMime\n' + 'FROM TestResultFiles,\n' + ' TestResultStrTab AS StrTabFile,\n' + ' TestResultStrTab AS StrTabDesc,\n' + ' TestResultStrTab AS StrTabKind,\n' + ' TestResultStrTab AS StrTabMime,\n' + ' TestResults\n' + 'WHERE TestResultFiles.idTestResultFile = %s\n' + ' AND TestResultFiles.idStrFile = StrTabFile.idStr\n' + ' AND TestResultFiles.idStrDescription = StrTabDesc.idStr\n' + ' AND TestResultFiles.idStrKind = StrTabKind.idStr\n' + ' AND TestResultFiles.idStrMime = StrTabMime.idStr\n' + ' AND TestResults.idTestResult = TestResultFiles.idTestResult\n' + ' AND TestResults.idTestSet = %s\n' + , ( idTestResultFile, idTestSet, )); + return TestResultFileDataEx().initFromDbRow(self._oDb.fetchOne()); + + + def getById(self, idTestSet): + """ + Get TestSet table record by its id + """ + self._oDb.execute('SELECT *\n' + 'FROM TestSets\n' + 'WHERE idTestSet=%s\n', + (idTestSet,)) + + aRows = self._oDb.fetchAll() + if len(aRows) not in (0, 1): + raise TMTooManyRows('Found more than one test sets with the same credentials. Database structure is corrupted.') + try: + return TestSetData().initFromDbRow(aRows[0]) + except IndexError: + return None + + + def fetchOrphaned(self): + """ + Returns a list of TestSetData objects of orphaned test sets. + + A test set is orphaned if tsDone is NULL and the testbox has created + one or more newer testsets. + """ + + self._oDb.execute('SELECT TestSets.*\n' + 'FROM TestSets,\n' + ' (SELECT idTestSet, idTestBox FROM TestSets WHERE tsDone is NULL) AS t\n' + 'WHERE TestSets.idTestSet = t.idTestSet\n' + ' AND EXISTS(SELECT 1 FROM TestSets st\n' + ' WHERE st.idTestBox = t.idTestBox AND st.idTestSet > t.idTestSet)\n' + ' AND NOT EXISTS(SELECT 1 FROM TestBoxStatuses tbs\n' + ' WHERE tbs.idTestBox = t.idTestBox AND tbs.idTestSet = t.idTestSet)\n' + 'ORDER by TestSets.idTestBox, TestSets.idTestSet' + ); + aoRet = []; + for aoRow in self._oDb.fetchAll(): + aoRet.append(TestSetData().initFromDbRow(aoRow)); + return aoRet; + + def fetchByAge(self, tsNow = None, cHoursBack = 24): + """ + Returns a list of TestSetData objects of a given time period (default is 24 hours). + + Returns None if no testsets stored, + Returns an empty list if no testsets found with given criteria. + """ + if tsNow is None: + tsNow = self._oDb.getCurrentTimestamp(); + + if self._oDb.getRowCount() == 0: + return None; + + self._oDb.execute('(SELECT *\n' + ' FROM TestSets\n' + ' WHERE tsDone <= %s\n' + ' AND tsDone > (%s - interval \'%s hours\')\n' + ')\n' + , ( tsNow, tsNow, cHoursBack, )); + + aoRet = []; + for aoRow in self._oDb.fetchAll(): + aoRet.append(TestSetData().initFromDbRow(aoRow)); + return aoRet; + + def isTestBoxExecutingTooRapidly(self, idTestBox): ## s/To/Too/ + """ + Checks whether the specified test box is executing tests too rapidly. + + The parameters defining too rapid execution are defined in config.py. + + Returns True if it does, False if it doesn't. + May raise database problems. + """ + + self._oDb.execute('(\n' + 'SELECT tsCreated\n' + 'FROM TestSets\n' + 'WHERE idTestBox = %s\n' + ' AND tsCreated >= (CURRENT_TIMESTAMP - interval \'%s seconds\')\n' + ') UNION (\n' + 'SELECT tsCreated\n' + 'FROM TestSets\n' + 'WHERE idTestBox = %s\n' + ' AND tsCreated >= (CURRENT_TIMESTAMP - interval \'%s seconds\')\n' + ' AND enmStatus >= \'failure\'\n' + ')' + , ( idTestBox, config.g_kcSecMinSinceLastTask, + idTestBox, config.g_kcSecMinSinceLastFailedTask, )); + return self._oDb.getRowCount() > 0; + + + # + # The virtual test sheriff interface. + # + + def fetchBadTestBoxIds(self, cHoursBack = 2, tsNow = None, aidFailureReasons = None): + """ + Fetches a list of test box IDs which returned bad-testbox statuses in the + given period (tsDone). + """ + if tsNow is None: + tsNow = self._oDb.getCurrentTimestamp(); + if aidFailureReasons is None: + aidFailureReasons = [ -1, ]; + self._oDb.execute('(SELECT idTestBox\n' + ' FROM TestSets\n' + ' WHERE TestSets.enmStatus = \'bad-testbox\'\n' + ' AND tsDone <= %s\n' + ' AND tsDone > (%s - interval \'%s hours\')\n' + ') UNION (\n' + ' SELECT TestSets.idTestBox\n' + ' FROM TestSets,\n' + ' TestResultFailures\n' + ' WHERE TestSets.tsDone <= %s\n' + ' AND TestSets.tsDone > (%s - interval \'%s hours\')\n' + ' AND TestSets.enmStatus >= \'failure\'::TestStatus_T\n' + ' AND TestSets.idTestSet = TestResultFailures.idTestSet\n' + ' AND TestResultFailures.tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND TestResultFailures.idFailureReason IN (' + + ', '.join([str(i) for i in aidFailureReasons]) + ')\n' + ')\n' + , ( tsNow, tsNow, cHoursBack, + tsNow, tsNow, cHoursBack, )); + return [aoRow[0] for aoRow in self._oDb.fetchAll()]; + + def fetchSetsForTestBox(self, idTestBox, cHoursBack = 2, tsNow = None): + """ + Fetches the TestSet rows for idTestBox for the given period (tsDone), w/o running ones. + + Returns list of TestSetData sorted by tsDone in descending order. + """ + if tsNow is None: + tsNow = self._oDb.getCurrentTimestamp(); + self._oDb.execute('SELECT *\n' + 'FROM TestSets\n' + 'WHERE TestSets.idTestBox = %s\n' + ' AND tsDone IS NOT NULL\n' + ' AND tsDone <= %s\n' + ' AND tsDone > (%s - interval \'%s hours\')\n' + 'ORDER by tsDone DESC\n' + , ( idTestBox, tsNow, tsNow, cHoursBack,)); + return self._dbRowsToModelDataList(TestSetData); + + def fetchFailedSetsWithoutReason(self, cHoursBack = 2, tsNow = None): + """ + Fetches the TestSet failure rows without any currently (CURRENT_TIMESTAMP + not tsNow) assigned failure reason. + + Returns list of TestSetData sorted by tsDone in descending order. + + Note! Includes bad-testbox sets too as it can be useful to analyze these + too even if we normally count them in the 'skipped' category. + """ + if tsNow is None: + tsNow = self._oDb.getCurrentTimestamp(); + self._oDb.execute('SELECT TestSets.*\n' + 'FROM TestSets\n' + ' LEFT OUTER JOIN TestResultFailures\n' + ' ON TestResultFailures.idTestSet = TestSets.idTestSet\n' + ' AND TestResultFailures.tsExpire = \'infinity\'::TIMESTAMP\n' + 'WHERE TestSets.tsDone IS NOT NULL\n' + ' AND TestSets.enmStatus IN ( %s, %s, %s, %s )\n' + ' AND TestSets.tsDone <= %s\n' + ' AND TestSets.tsDone > (%s - interval \'%s hours\')\n' + ' AND TestResultFailures.idTestSet IS NULL\n' + 'ORDER by tsDone DESC\n' + , ( TestSetData.ksTestStatus_Failure, TestSetData.ksTestStatus_TimedOut, + TestSetData.ksTestStatus_Rebooted, TestSetData.ksTestStatus_BadTestBox, + tsNow, + tsNow, cHoursBack,)); + return self._dbRowsToModelDataList(TestSetData); + + + +# +# Unit testing. +# + +# pylint: disable=missing-docstring +class TestSetDataTestCase(ModelDataBaseTestCase): + def setUp(self): + self.aoSamples = [TestSetData(),]; + +if __name__ == '__main__': + unittest.main(); + # not reached. diff --git a/src/VBox/ValidationKit/testmanager/core/useraccount.pgsql b/src/VBox/ValidationKit/testmanager/core/useraccount.pgsql new file mode 100644 index 00000000..1abd8b71 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/useraccount.pgsql @@ -0,0 +1,178 @@ +-- $Id: useraccount.pgsql $ +--- @file +-- VBox Test Manager Database Stored Procedures - UserAccounts. +-- + +-- +-- Copyright (C) 2012-2023 Oracle and/or its affiliates. +-- +-- This file is part of VirtualBox base platform packages, as +-- available from https://www.virtualbox.org. +-- +-- This program is free software; you can redistribute it and/or +-- modify it under the terms of the GNU General Public License +-- as published by the Free Software Foundation, in version 3 of the +-- License. +-- +-- This program is distributed in the hope that it will be useful, but +-- WITHOUT ANY WARRANTY; without even the implied warranty of +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +-- General Public License for more details. +-- +-- You should have received a copy of the GNU General Public License +-- along with this program; if not, see <https://www.gnu.org/licenses>. +-- +-- The contents of this file may alternatively be used under the terms +-- of the Common Development and Distribution License Version 1.0 +-- (CDDL), a copy of it is provided in the "COPYING.CDDL" file included +-- in the VirtualBox distribution, in which case the provisions of the +-- CDDL are applicable instead of those of the GPL. +-- +-- You may elect to license modified versions of this file under the +-- terms and conditions of either the GPL or the CDDL or both. +-- +-- SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +-- + +\set ON_ERROR_STOP 1 +\connect testmanager; + +--- +-- Checks if the user name and login name are unique, ignoring a_uidIgnore. +-- Raises exception if duplicates are found. +-- +-- @internal +-- +CREATE OR REPLACE FUNCTION UserAccountLogic_checkUniqueUser(a_sUsername TEXT, a_sLoginName TEXT, a_uidIgnore INTEGER) + RETURNS VOID AS $$ + DECLARE + v_cRows INTEGER; + BEGIN + -- sUserName + SELECT COUNT(*) INTO v_cRows + FROM Users + WHERE sUsername = a_sUsername + AND tsExpire = 'infinity'::TIMESTAMP + AND uid <> a_uidIgnore; + IF v_cRows <> 0 THEN + RAISE EXCEPTION 'Duplicate user name "%" (% times)', a_sUsername, v_cRows; + END IF; + + -- sLoginName + SELECT COUNT(*) INTO v_cRows + FROM Users + WHERE sLoginName = a_sLoginName + AND tsExpire = 'infinity'::TIMESTAMP + AND uid <> a_uidIgnore; + IF v_cRows <> 0 THEN + RAISE EXCEPTION 'Duplicate login name "%" (% times)', a_sUsername, v_cRows; + END IF; + END; +$$ LANGUAGE plpgsql; + +--- +-- Check that the user account exists. +-- Raises exception if it doesn't. +-- +-- @internal +-- +CREATE OR REPLACE FUNCTION UserAccountLogic_checkExists(a_uid INTEGER) RETURNS VOID AS $$ + DECLARE + v_cUpdatedRows INTEGER; + BEGIN + IF NOT EXISTS( SELECT * + FROM Users + WHERE uid = a_uid + AND tsExpire = 'infinity'::TIMESTAMP ) THEN + RAISE EXCEPTION 'User with ID % does not currently exist', a_uid; + END IF; + END; +$$ LANGUAGE plpgsql; + +--- +-- Historize a row. +-- @internal +-- +CREATE OR REPLACE FUNCTION UserAccountLogic_historizeEntry(a_uid INTEGER, a_tsExpire TIMESTAMP WITH TIME ZONE) + RETURNS VOID AS $$ + DECLARE + v_cUpdatedRows INTEGER; + BEGIN + UPDATE Users + SET tsExpire = a_tsExpire + WHERE uid = a_uid + AND tsExpire = 'infinity'::TIMESTAMP; + GET DIAGNOSTICS v_cUpdatedRows = ROW_COUNT; + IF v_cUpdatedRows <> 1 THEN + IF v_cUpdatedRows = 0 THEN + RAISE EXCEPTION 'User with ID % does not currently exist', a_uid; + END IF; + RAISE EXCEPTION 'Integrity error in UserAccounts: % current rows with uid=%d', v_cUpdatedRows, a_uid; + END IF; + END; +$$ LANGUAGE plpgsql; + + +--- +-- Adds a new user. +-- +CREATE OR REPLACE FUNCTION UserAccountLogic_addEntry(a_uidAuthor INTEGER, a_sUsername TEXT, a_sEmail TEXT, a_sFullName TEXT, + a_sLoginName TEXT, a_fReadOnly BOOLEAN) + RETURNS VOID AS $$ + DECLARE + v_cRows INTEGER; + BEGIN + PERFORM UserAccountLogic_checkUniqueUser(a_sUsername, a_sLoginName, -1); + INSERT INTO Users(uidAuthor, sUsername, sEmail, sFullName, sLoginName) + VALUES (a_uidAuthor, a_sUsername, a_sEmail, a_sFullName, a_sLoginName); + END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION UserAccountLogic_editEntry(a_uidAuthor INTEGER, a_uid INTEGER, a_sUsername TEXT, a_sEmail TEXT, + a_sFullName TEXT, a_sLoginName TEXT, a_fReadOnly BOOLEAN) + RETURNS VOID AS $$ + BEGIN + PERFORM UserAccountLogic_checkExists(a_uid); + PERFORM UserAccountLogic_checkUniqueUser(a_sUsername, a_sLoginName, a_uid); + + PERFORM UserAccountLogic_historizeEntry(a_uid, CURRENT_TIMESTAMP); + INSERT INTO Users (uid, uidAuthor, sUsername, sEmail, sFullName, sLoginName, fReadOnly) + VALUES (a_uid, a_uidAuthor, a_sUsername, a_sEmail, a_sFullName, a_sLoginName, a_fReadOnly); + END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION UserAccountLogic_delEntry(a_uidAuthor INTEGER, a_uid INTEGER) RETURNS VOID AS $$ + DECLARE + v_Row Users%ROWTYPE; + v_tsEffective TIMESTAMP WITH TIME ZONE; + BEGIN + -- + -- To preserve the information about who deleted the record, we try to + -- add a dummy record which expires immediately. I say try because of + -- the primary key, we must let the new record be valid for 1 us. :-( + -- + + SELECT * INTO STRICT v_Row + FROM Users + WHERE uid = a_uid + AND tsExpire = 'infinity'::TIMESTAMP; + + v_tsEffective := CURRENT_TIMESTAMP - INTERVAL '1 microsecond'; + IF v_Row.tsEffective < v_tsEffective THEN + PERFORM UserAccountLogic_historizeEntry(a_uid, v_tsEffective); + v_Row.tsEffective = v_tsEffective; + v_Row.tsExpire = CURRENT_TIMESTAMP; + INSERT INTO Users VALUES (v_Row.*); + ELSE + PERFORM UserAccountLogic_historizeEntry(a_uid, CURRENT_TIMESTAMP); + END IF; + + EXCEPTION + WHEN NO_DATA_FOUND THEN + RAISE EXCEPTION 'User with ID % does not currently exist', a_uid; + WHEN TOO_MANY_ROWS THEN + RAISE EXCEPTION 'Integrity error in UserAccounts: Too many current rows for %', a_uid; + END; +$$ LANGUAGE plpgsql; + diff --git a/src/VBox/ValidationKit/testmanager/core/useraccount.py b/src/VBox/ValidationKit/testmanager/core/useraccount.py new file mode 100755 index 00000000..6c3f3f51 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/useraccount.py @@ -0,0 +1,302 @@ +# -*- coding: utf-8 -*- +# $Id: useraccount.py $ + +""" +Test Manager - User DB records management. +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Standard python imports. +import unittest; + +# Validation Kit imports. +from testmanager import config; +from testmanager.core.base import ModelDataBase, ModelLogicBase, ModelDataBaseTestCase, TMTooManyRows, TMRowNotFound; + + +class UserAccountData(ModelDataBase): + """ + User account data + """ + + ksIdAttr = 'uid'; + + ksParam_uid = 'UserAccount_uid' + ksParam_tsExpire = 'UserAccount_tsExpire' + ksParam_tsEffective = 'UserAccount_tsEffective' + ksParam_uidAuthor = 'UserAccount_uidAuthor' + ksParam_sLoginName = 'UserAccount_sLoginName' + ksParam_sUsername = 'UserAccount_sUsername' + ksParam_sEmail = 'UserAccount_sEmail' + ksParam_sFullName = 'UserAccount_sFullName' + ksParam_fReadOnly = 'UserAccount_fReadOnly' + + kasAllowNullAttributes = ['uid', 'tsEffective', 'tsExpire', 'uidAuthor']; + + + def __init__(self): + """Init parameters""" + ModelDataBase.__init__(self); + self.uid = None; + self.tsEffective = None; + self.tsExpire = None; + self.uidAuthor = None; + self.sUsername = None; + self.sEmail = None; + self.sFullName = None; + self.sLoginName = None; + self.fReadOnly = None; + + def initFromDbRow(self, aoRow): + """ + Init from database table row + Returns self. Raises exception of the row is None. + """ + if aoRow is None: + raise TMRowNotFound('User not found.'); + + self.uid = aoRow[0]; + self.tsEffective = aoRow[1]; + self.tsExpire = aoRow[2]; + self.uidAuthor = aoRow[3]; + self.sUsername = aoRow[4]; + self.sEmail = aoRow[5]; + self.sFullName = aoRow[6]; + self.sLoginName = aoRow[7]; + self.fReadOnly = aoRow[8]; + return self; + + def initFromDbWithId(self, oDb, uid, tsNow = None, sPeriodBack = None): + """ + Initialize the object from the database. + """ + oDb.execute(self.formatSimpleNowAndPeriodQuery(oDb, + 'SELECT *\n' + 'FROM Users\n' + 'WHERE uid = %s\n' + , ( uid, ), tsNow, sPeriodBack)); + aoRow = oDb.fetchOne() + if aoRow is None: + raise TMRowNotFound('uid=%s not found (tsNow=%s sPeriodBack=%s)' % (uid, tsNow, sPeriodBack,)); + return self.initFromDbRow(aoRow); + + def _validateAndConvertAttribute(self, sAttr, sParam, oValue, aoNilValues, fAllowNull, oDb): + # Custom handling of the email field. + if sAttr == 'sEmail': + return ModelDataBase.validateEmail(oValue, aoNilValues = aoNilValues, fAllowNull = fAllowNull); + + # Automatically lowercase the login name if we're supposed to do case + # insensitive matching. (The feature assumes lower case in DB.) + if sAttr == 'sLoginName' and oValue is not None and config.g_kfLoginNameCaseInsensitive: + oValue = oValue.lower(); + + return ModelDataBase._validateAndConvertAttribute(self, sAttr, sParam, oValue, aoNilValues, fAllowNull, oDb); + + +class UserAccountLogic(ModelLogicBase): + """ + User account logic (for the Users table). + """ + + def __init__(self, oDb): + ModelLogicBase.__init__(self, oDb) + self.dCache = None; + + def fetchForListing(self, iStart, cMaxRows, tsNow, aiSortColumns = None): + """ + Fetches user accounts. + + Returns an array (list) of UserAccountData items, empty list if none. + Raises exception on error. + """ + _ = aiSortColumns; + if tsNow is None: + self._oDb.execute('SELECT *\n' + 'FROM Users\n' + 'WHERE tsExpire = \'infinity\'::TIMESTAMP\n' + 'ORDER BY sUsername DESC\n' + 'LIMIT %s OFFSET %s\n' + , (cMaxRows, iStart,)); + else: + self._oDb.execute('SELECT *\n' + 'FROM Users\n' + 'WHERE tsExpire > %s\n' + ' AND tsEffective <= %s\n' + 'ORDER BY sUsername DESC\n' + 'LIMIT %s OFFSET %s\n' + , (tsNow, tsNow, cMaxRows, iStart,)); + + aoRows = []; + for _ in range(self._oDb.getRowCount()): + aoRows.append(UserAccountData().initFromDbRow(self._oDb.fetchOne())); + return aoRows; + + def addEntry(self, oData, uidAuthor, fCommit = False): + """ + Add user account entry to the DB. + """ + self._oDb.callProc('UserAccountLogic_addEntry', + (uidAuthor, oData.sUsername, oData.sEmail, oData.sFullName, oData.sLoginName, oData.fReadOnly)); + self._oDb.maybeCommit(fCommit); + return True; + + def editEntry(self, oData, uidAuthor, fCommit = False): + """ + Modify user account. + """ + self._oDb.callProc('UserAccountLogic_editEntry', + ( uidAuthor, oData.uid, oData.sUsername, oData.sEmail, + oData.sFullName, oData.sLoginName, oData.fReadOnly)); + self._oDb.maybeCommit(fCommit); + return True; + + def removeEntry(self, uidAuthor, uid, fCascade = False, fCommit = False): + """ + Delete user account + """ + self._oDb.callProc('UserAccountLogic_delEntry', (uidAuthor, uid)); + self._oDb.maybeCommit(fCommit); + _ = fCascade; + return True; + + def _getByField(self, sField, sValue): + """ + Get user account record by its field value + """ + self._oDb.execute('SELECT *\n' + 'FROM Users\n' + 'WHERE tsExpire = \'infinity\'::TIMESTAMP\n' + ' AND ' + sField + ' = %s' + , (sValue,)) + + aRows = self._oDb.fetchAll() + if len(aRows) not in (0, 1): + raise TMTooManyRows('Found more than one user account with the same credentials. Database structure is corrupted.') + + try: + return aRows[0] + except IndexError: + return [] + + def getById(self, idUserId): + """ + Get user account information by ID. + """ + return self._getByField('uid', idUserId) + + def tryFetchAccountByLoginName(self, sLoginName): + """ + Try get user account information by login name. + + Returns UserAccountData if found, None if not. + Raises exception on DB error. + """ + if config.g_kfLoginNameCaseInsensitive: + sLoginName = sLoginName.lower(); + + self._oDb.execute('SELECT *\n' + 'FROM Users\n' + 'WHERE sLoginName = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , (sLoginName, )); + if self._oDb.getRowCount() != 1: + if self._oDb.getRowCount() != 0: + raise self._oDb.integrityException('%u rows in Users with sLoginName="%s"' + % (self._oDb.getRowCount(), sLoginName)); + return None; + return UserAccountData().initFromDbRow(self._oDb.fetchOne()); + + def cachedLookup(self, uid): + """ + Looks up the current UserAccountData object for uid via an object cache. + + Returns a shared UserAccountData object. None if not found. + Raises exception on DB error. + """ + if self.dCache is None: + self.dCache = self._oDb.getCache('UserAccount'); + + oUser = self.dCache.get(uid, None); + if oUser is None: + self._oDb.execute('SELECT *\n' + 'FROM Users\n' + 'WHERE uid = %s\n' + ' AND tsExpire = \'infinity\'::TIMESTAMP\n' + , (uid, )); + if self._oDb.getRowCount() == 0: + # Maybe it was deleted, try get the last entry. + self._oDb.execute('SELECT *\n' + 'FROM Users\n' + 'WHERE uid = %s\n' + 'ORDER BY tsExpire DESC\n' + 'LIMIT 1\n' + , (uid, )); + elif self._oDb.getRowCount() > 1: + raise self._oDb.integrityException('%s infinity rows for %s' % (self._oDb.getRowCount(), uid)); + + if self._oDb.getRowCount() == 1: + oUser = UserAccountData().initFromDbRow(self._oDb.fetchOne()); + self.dCache[uid] = oUser; + return oUser; + + def resolveChangeLogAuthors(self, aoEntries): + """ + Given an array of ChangeLogEntry instances, set sAuthor to whatever + uidAuthor resolves to. + + Returns aoEntries. + Raises exception on DB error. + """ + for oEntry in aoEntries: + oUser = self.cachedLookup(oEntry.uidAuthor) + if oUser is not None: + oEntry.sAuthor = oUser.sUsername; + return aoEntries; + + +# +# Unit testing. +# + +# pylint: disable=missing-docstring +class UserAccountDataTestCase(ModelDataBaseTestCase): + def setUp(self): + self.aoSamples = [UserAccountData(),]; + +if __name__ == '__main__': + unittest.main(); + # not reached. + diff --git a/src/VBox/ValidationKit/testmanager/core/vcsbugreference.py b/src/VBox/ValidationKit/testmanager/core/vcsbugreference.py new file mode 100755 index 00000000..9ae9c941 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/vcsbugreference.py @@ -0,0 +1,251 @@ +# -*- coding: utf-8 -*- +# $Id: vcsbugreference.py $ + +""" +Test Manager - VcsBugReferences +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Standard python imports. +import unittest; + +# Validation Kit imports. +from testmanager.core.base import ModelDataBase, ModelDataBaseTestCase, ModelLogicBase, TMExceptionBase; + + +class VcsBugReferenceData(ModelDataBase): + """ + A version control system (VCS) bug tracker reference (commit message tag). + """ + + #kasIdAttr = ['sRepository','iRevision', 'sBugTracker', 'iBugNo']; + + ksParam_sRepository = 'VcsBugReference_sRepository'; + ksParam_iRevision = 'VcsBugReference_iRevision'; + ksParam_sBugTracker = 'VcsBugReference_sBugTracker'; + ksParam_lBugNo = 'VcsBugReference_lBugNo'; + + kasAllowNullAttributes = [ ]; + + def __init__(self): + ModelDataBase.__init__(self); + + # + # Initialize with defaults. + # See the database for explanations of each of these fields. + # + self.sRepository = None; + self.iRevision = None; + self.sBugTracker = None; + self.lBugNo = None; + + def initFromDbRow(self, aoRow): + """ + Re-initializes the object from a SELECT * FROM VcsBugReferences row. + Returns self. Raises exception if aoRow is None. + """ + if aoRow is None: + raise TMExceptionBase('VcsBugReference not found.'); + + self.sRepository = aoRow[0]; + self.iRevision = aoRow[1]; + self.sBugTracker = aoRow[2]; + self.lBugNo = aoRow[3]; + return self; + + def initFromValues(self, sRepository, iRevision, sBugTracker, lBugNo): + """ + Reinitializes form a set of values. + return self. + """ + self.sRepository = sRepository; + self.iRevision = iRevision; + self.sBugTracker = sBugTracker; + self.lBugNo = lBugNo; + return self; + + +class VcsBugReferenceDataEx(VcsBugReferenceData): + """ + Extended version of VcsBugReferenceData that includes the commit details. + """ + def __init__(self): + VcsBugReferenceData.__init__(self); + self.tsCreated = None; + self.sAuthor = None; + self.sMessage = None; + + def initFromDbRow(self, aoRow): + VcsBugReferenceData.initFromDbRow(self, aoRow); + self.tsCreated = aoRow[4]; + self.sAuthor = aoRow[5]; + self.sMessage = aoRow[6]; + return self; + + +class VcsBugReferenceLogic(ModelLogicBase): # pylint: disable=too-few-public-methods + """ + VCS revision <-> bug tracker references database logic. + """ + + # + # Standard methods. + # + + def fetchForListing(self, iStart, cMaxRows, tsNow, aiSortColumns = None): + """ + Fetches VCS revisions for listing. + + Returns an array (list) of VcsBugReferenceData items, empty list if none. + Raises exception on error. + """ + _ = tsNow; _ = aiSortColumns; + self._oDb.execute(''' +SELECT * +FROM VcsBugReferences +ORDER BY sRepository, iRevision, sBugTracker, lBugNo +LIMIT %s OFFSET %s +''', (cMaxRows, iStart,)); + + aoRows = []; + for _ in range(self._oDb.getRowCount()): + aoRows.append(VcsBugReferenceData().initFromDbRow(self._oDb.fetchOne())); + return aoRows; + + def exists(self, oData): + """ + Checks if the data is already present in the DB. + Returns True / False. + Raises exception on input and database errors. + """ + self._oDb.execute(''' +SELECT COUNT(*) +FROM VcsBugReferences +WHERE sRepository = %s + AND iRevision = %s + AND sBugTracker = %s + AND lBugNo = %s +''', ( oData.sRepository, oData.iRevision, oData.sBugTracker, oData.lBugNo)); + cRows = self._oDb.fetchOne()[0]; + if cRows < 0 or cRows > 1: + raise TMExceptionBase('VcsBugReferences has a primary key problem: %u duplicates' % (cRows,)); + return cRows != 0; + + + # + # Other methods. + # + + def addVcsBugReference(self, oData, fCommit = False): + """ + Adds (or updates) a tree revision record. + Raises exception on input and database errors. + """ + + # Check VcsBugReferenceData before do anything + dDataErrors = oData.validateAndConvert(self._oDb, oData.ksValidateFor_Add); + if dDataErrors: + raise TMExceptionBase('Invalid data passed to addVcsBugReference(): %s' % (dDataErrors,)); + + # Does it already exist? + if not self.exists(oData): + # New row. + self._oDb.execute('INSERT INTO VcsBugReferences (sRepository, iRevision, sBugTracker, lBugNo)\n' + 'VALUES (%s, %s, %s, %s)\n' + , ( oData.sRepository, + oData.iRevision, + oData.sBugTracker, + oData.lBugNo, + )); + + self._oDb.maybeCommit(fCommit); + return oData; + + def getLastRevision(self, sRepository): + """ + Get the last known revision number for the given repository, returns 0 + if the repository is not known to us: + """ + self._oDb.execute(''' +SELECT iRevision +FROM VcsBugReferences +WHERE sRepository = %s +ORDER BY iRevision DESC +LIMIT 1 +''', ( sRepository, )); + if self._oDb.getRowCount() == 0: + return 0; + return self._oDb.fetchOne()[0]; + + def fetchForBug(self, sBugTracker, lBugNo): + """ + Fetches VCS revisions for a bug. + + Returns an array (list) of VcsBugReferenceDataEx items, empty list if none. + Raises exception on error. + """ + self._oDb.execute(''' +SELECT VcsBugReferences.*, + VcsRevisions.tsCreated, + VcsRevisions.sAuthor, + VcsRevisions.sMessage +FROM VcsBugReferences +LEFT OUTER JOIN VcsRevisions ON ( VcsRevisions.sRepository = VcsBugReferences.sRepository + AND VcsRevisions.iRevision = VcsBugReferences.iRevision ) +WHERE sBugTracker = %s + AND lBugNo = %s +ORDER BY VcsRevisions.tsCreated, VcsBugReferences.sRepository, VcsBugReferences.iRevision +''', (sBugTracker, lBugNo,)); + + aoRows = []; + for _ in range(self._oDb.getRowCount()): + aoRows.append(VcsBugReferenceDataEx().initFromDbRow(self._oDb.fetchOne())); + return aoRows; + + +# +# Unit testing. +# + +# pylint: disable=missing-docstring +class VcsBugReferenceDataTestCase(ModelDataBaseTestCase): + def setUp(self): + self.aoSamples = [VcsBugReferenceData(),]; + +if __name__ == '__main__': + unittest.main(); + # not reached. + diff --git a/src/VBox/ValidationKit/testmanager/core/vcsrevisions.py b/src/VBox/ValidationKit/testmanager/core/vcsrevisions.py new file mode 100755 index 00000000..ea841d11 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/vcsrevisions.py @@ -0,0 +1,254 @@ +# -*- coding: utf-8 -*- +# $Id: vcsrevisions.py $ + +""" +Test Manager - VcsRevisions +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Standard python imports. +import unittest; + +# Validation Kit imports. +from testmanager.core.base import ModelDataBase, ModelDataBaseTestCase, ModelLogicBase, TMExceptionBase; + + +class VcsRevisionData(ModelDataBase): + """ + A version control system (VCS) revision. + """ + + #kasIdAttr = ['sRepository',iRevision]; + + ksParam_sRepository = 'VcsRevision_sRepository'; + ksParam_iRevision = 'VcsRevision_iRevision'; + ksParam_tsCreated = 'VcsRevision_tsCreated'; + ksParam_sAuthor = 'VcsRevision_sAuthor'; + ksParam_sMessage = 'VcsRevision_sMessage'; + + kasAllowNullAttributes = [ ]; + kfAllowUnicode_sMessage = True; + kcchMax_sMessage = 8192; + + def __init__(self): + ModelDataBase.__init__(self); + + # + # Initialize with defaults. + # See the database for explanations of each of these fields. + # + self.sRepository = None; + self.iRevision = None; + self.tsCreated = None; + self.sAuthor = None; + self.sMessage = None; + + def initFromDbRow(self, aoRow): + """ + Re-initializes the object from a SELECT * FROM VcsRevisions row. + Returns self. Raises exception if aoRow is None. + """ + if aoRow is None: + raise TMExceptionBase('VcsRevision not found.'); + + self.sRepository = aoRow[0]; + self.iRevision = aoRow[1]; + self.tsCreated = aoRow[2]; + self.sAuthor = aoRow[3]; + self.sMessage = aoRow[4]; + return self; + + def initFromDbWithRepoAndRev(self, oDb, sRepository, iRevision): + """ + Initialize from the database, given the tree and revision of a row. + """ + oDb.execute('SELECT * FROM VcsRevisions WHERE sRepository = %s AND iRevision = %u', (sRepository, iRevision,)); + aoRow = oDb.fetchOne() + if aoRow is None: + raise TMExceptionBase('sRepository = %s iRevision = %u not found' % (sRepository, iRevision, )); + return self.initFromDbRow(aoRow); + + def initFromValues(self, sRepository, iRevision, tsCreated, sAuthor, sMessage): + """ + Reinitializes form a set of values. + return self. + """ + self.sRepository = sRepository; + self.iRevision = iRevision; + self.tsCreated = tsCreated; + self.sAuthor = sAuthor; + self.sMessage = sMessage; + return self; + + +class VcsRevisionLogic(ModelLogicBase): # pylint: disable=too-few-public-methods + """ + VCS revisions database logic. + """ + + # + # Standard methods. + # + + def fetchForListing(self, iStart, cMaxRows, tsNow, aiSortColumns = None): + """ + Fetches VCS revisions for listing. + + Returns an array (list) of VcsRevisionData items, empty list if none. + Raises exception on error. + """ + _ = tsNow; _ = aiSortColumns; + self._oDb.execute('SELECT *\n' + 'FROM VcsRevisions\n' + 'ORDER BY tsCreated, sRepository, iRevision\n' + 'LIMIT %s OFFSET %s\n' + , (cMaxRows, iStart,)); + + aoRows = []; + for _ in range(self._oDb.getRowCount()): + aoRows.append(VcsRevisionData().initFromDbRow(self._oDb.fetchOne())); + return aoRows; + + def tryFetch(self, sRepository, iRevision): + """ + Tries to fetch the specified tree revision record. + Returns VcsRevisionData instance if found, None if not found. + Raises exception on input and database errors. + """ + self._oDb.execute('SELECT * FROM VcsRevisions WHERE sRepository = %s AND iRevision = %s', + ( sRepository, iRevision, )); + aaoRows = self._oDb.fetchAll(); + if len(aaoRows) == 1: + return VcsRevisionData().initFromDbRow(aaoRows[0]); + if aaoRows: + raise TMExceptionBase('VcsRevisions has a primary key problem: %u duplicates' % (len(aaoRows),)); + return None + + + # + # Other methods. + # + + def addVcsRevision(self, oData, fCommit = False): + """ + Adds (or updates) a tree revision record. + Raises exception on input and database errors. + """ + + # Check VcsRevisionData before do anything + dDataErrors = oData.validateAndConvert(self._oDb, oData.ksValidateFor_Add); + if dDataErrors: + raise TMExceptionBase('Invalid data passed to addVcsRevision(): %s' % (dDataErrors,)); + + # Does it already exist? + oOldData = self.tryFetch(oData.sRepository, oData.iRevision); + if oOldData is None: + # New row. + self._oDb.execute('INSERT INTO VcsRevisions (sRepository, iRevision, tsCreated, sAuthor, sMessage)\n' + 'VALUES (%s, %s, %s, %s, %s)\n' + , ( oData.sRepository, + oData.iRevision, + oData.tsCreated, + oData.sAuthor, + oData.sMessage, + )); + elif not oOldData.isEqual(oData): + # Update old row. + self._oDb.execute('UPDATE VcsRevisions\n' + ' SET tsCreated = %s,\n' + ' sAuthor = %s,\n' + ' sMessage = %s\n' + 'WHERE sRepository = %s\n' + ' AND iRevision = %s' + , ( oData.tsCreated, + oData.sAuthor, + oData.sMessage, + oData.sRepository, + oData.iRevision, + )); + + self._oDb.maybeCommit(fCommit); + return oData; + + def getLastRevision(self, sRepository): + """ + Get the last known revision number for the given repository, returns 0 + if the repository is not known to us: + """ + self._oDb.execute('SELECT iRevision\n' + 'FROM VcsRevisions\n' + 'WHERE sRepository = %s\n' + 'ORDER BY iRevision DESC\n' + 'LIMIT 1\n' + , ( sRepository, )); + if self._oDb.getRowCount() == 0: + return 0; + return self._oDb.fetchOne()[0]; + + def fetchTimeline(self, sRepository, iRevision, cEntriesBack): + """ + Fetches a VCS timeline portion for a repository. + + Returns an array (list) of VcsRevisionData items, empty list if none. + Raises exception on error. + """ + self._oDb.execute('SELECT *\n' + 'FROM VcsRevisions\n' + 'WHERE sRepository = %s\n' + ' AND iRevision > %s\n' + ' AND iRevision <= %s\n' + 'ORDER BY iRevision DESC\n' + 'LIMIT %s\n' + , ( sRepository, iRevision - cEntriesBack*2 + 1, iRevision, cEntriesBack)); + aoRows = []; + for _ in range(self._oDb.getRowCount()): + aoRows.append(VcsRevisionData().initFromDbRow(self._oDb.fetchOne())); + return aoRows; + + +# +# Unit testing. +# + +# pylint: disable=missing-docstring +class VcsRevisionDataTestCase(ModelDataBaseTestCase): + def setUp(self): + self.aoSamples = [VcsRevisionData(),]; + +if __name__ == '__main__': + unittest.main(); + # not reached. + diff --git a/src/VBox/ValidationKit/testmanager/core/webservergluebase.py b/src/VBox/ValidationKit/testmanager/core/webservergluebase.py new file mode 100755 index 00000000..a36e1af7 --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/webservergluebase.py @@ -0,0 +1,717 @@ +# -*- coding: utf-8 -*- +# $Id: webservergluebase.py $ + +""" +Test Manager Core - Web Server Abstraction Base Class. +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Standard python imports. +import cgitb +import codecs; +import os +import sys + +# Validation Kit imports. +from common import webutils, utils; +from testmanager import config; + + +class WebServerGlueException(Exception): + """ + For exceptions raised by glue code. + """ + pass; # pylint: disable=unnecessary-pass + + +class WebServerGlueBase(object): + """ + Web server interface abstraction and some HTML utils. + """ + + ## Enables more debug output. + kfDebugInfoEnabled = True; + + ## The maximum number of characters to cache. + kcchMaxCached = 65536; + + ## Special getUserName return value. + ksUnknownUser = 'Unknown User'; + + ## HTTP status codes and their messages. + kdStatusMsgs = { + 100: 'Continue', + 101: 'Switching Protocols', + 102: 'Processing', + 103: 'Early Hints', + 200: 'OK', + 201: 'Created', + 202: 'Accepted', + 203: 'Non-Authoritative Information', + 204: 'No Content', + 205: 'Reset Content', + 206: 'Partial Content', + 207: 'Multi-Status', + 208: 'Already Reported', + 226: 'IM Used', + 300: 'Multiple Choices', + 301: 'Moved Permantently', + 302: 'Found', + 303: 'See Other', + 304: 'Not Modified', + 305: 'Use Proxy', + 306: 'Switch Proxy', + 307: 'Temporary Redirect', + 308: 'Permanent Redirect', + 400: 'Bad Request', + 401: 'Unauthorized', + 402: 'Payment Required', + 403: 'Forbidden', + 404: 'Not Found', + 405: 'Method Not Allowed', + 406: 'Not Acceptable', + 407: 'Proxy Authentication Required', + 408: 'Request Timeout', + 409: 'Conflict', + 410: 'Gone', + 411: 'Length Required', + 412: 'Precondition Failed', + 413: 'Payload Too Large', + 414: 'URI Too Long', + 415: 'Unsupported Media Type', + 416: 'Range Not Satisfiable', + 417: 'Expectation Failed', + 418: 'I\'m a teapot', + 421: 'Misdirection Request', + 422: 'Unprocessable Entity', + 423: 'Locked', + 424: 'Failed Dependency', + 425: 'Too Early', + 426: 'Upgrade Required', + 428: 'Precondition Required', + 429: 'Too Many Requests', + 431: 'Request Header Fields Too Large', + 451: 'Unavailable For Legal Reasons', + 500: 'Internal Server Error', + 501: 'Not Implemented', + 502: 'Bad Gateway', + 503: 'Service Unavailable', + 504: 'Gateway Timeout', + 505: 'HTTP Version Not Supported', + 506: 'Variant Also Negotiates', + 507: 'Insufficient Storage', + 508: 'Loop Detected', + 510: 'Not Extended', + 511: 'Network Authentication Required', + }; + + + def __init__(self, sValidationKitDir, fHtmlDebugOutput = True): + self._sValidationKitDir = sValidationKitDir; + + # Debug + self.tsStart = utils.timestampNano(); + self._fHtmlDebugOutput = fHtmlDebugOutput; # For trace + self._oDbgFile = sys.stderr; + if config.g_ksSrvGlueDebugLogDst is not None and config.g_kfSrvGlueDebug is True: + self._oDbgFile = open(config.g_ksSrvGlueDebugLogDst, 'a'); # pylint: disable=consider-using-with,unspecified-encoding + if config.g_kfSrvGlueCgiDumpArgs: + self._oDbgFile.write('Arguments: %s\nEnvironment:\n' % (sys.argv,)); + if config.g_kfSrvGlueCgiDumpEnv: + for sVar in sorted(os.environ): + self._oDbgFile.write(' %s=\'%s\' \\\n' % (sVar, os.environ[sVar],)); + + self._afnDebugInfo = []; + + # HTTP header. + self._fHeaderWrittenOut = False; + self._dHeaderFields = \ + { \ + 'Content-Type': 'text/html; charset=utf-8', + }; + + # Body. + self._sBodyType = None; + self._dParams = {}; + self._sHtmlBody = ''; + self._cchCached = 0; + self._cchBodyWrittenOut = 0; + + # Output. + if sys.version_info[0] >= 3: + self.oOutputRaw = sys.stdout.detach(); # pylint: disable=no-member + sys.stdout = None; # Prevents flush_std_files() from complaining on stderr during sys.exit(). + else: + self.oOutputRaw = sys.stdout; + self.oOutputText = codecs.getwriter('utf-8')(self.oOutputRaw); + + + # + # Get stuff. + # + + def getParameters(self): + """ + Returns a dictionary with the query parameters. + + The parameter name is the key, the values are given as lists. If a + parameter is given more than once, the value is appended to the + existing dictionary entry. + """ + return {}; + + def getClientAddr(self): + """ + Returns the client address, as a string. + """ + raise WebServerGlueException('getClientAddr is not implemented'); + + def getMethod(self): + """ + Gets the HTTP request method. + """ + return 'POST'; + + def getLoginName(self): + """ + Gets login name provided by Apache. + Returns kUnknownUser if not logged on. + """ + return WebServerGlueBase.ksUnknownUser; + + def getUrlScheme(self): + """ + Gets scheme name (aka. access protocol) from request URL, i.e. 'http' or 'https'. + See also urlparse.scheme. + """ + return 'http'; + + def getUrlNetLoc(self): + """ + Gets the network location (server host name / ip) from the request URL. + See also urlparse.netloc. + """ + raise WebServerGlueException('getUrlNetLoc is not implemented'); + + def getUrlPath(self): + """ + Gets the hirarchical path (relative to server) from the request URL. + See also urlparse.path. + Note! This includes the leading slash. + """ + raise WebServerGlueException('getUrlPath is not implemented'); + + def getUrlBasePath(self): + """ + Gets the hirarchical base path (relative to server) from the request URL. + Note! This includes both a leading an trailing slash. + """ + sPath = self.getUrlPath(); # virtual method # pylint: disable=assignment-from-no-return + iLastSlash = sPath.rfind('/'); + if iLastSlash >= 0: + sPath = sPath[:iLastSlash]; + sPath = sPath.rstrip('/'); + return sPath + '/'; + + def getUrl(self): + """ + Gets the URL being accessed, sans parameters. + For instance this will return, "http://localhost/testmanager/admin.cgi" + when "http://localhost/testmanager/admin.cgi?blah=blah" is being access. + """ + return '%s://%s%s' % (self.getUrlScheme(), self.getUrlNetLoc(), self.getUrlPath()); + + def getBaseUrl(self): + """ + Gets the base URL (with trailing slash). + For instance this will return, "http://localhost/testmanager/" when + "http://localhost/testmanager/admin.cgi?blah=blah" is being access. + """ + return '%s://%s%s' % (self.getUrlScheme(), self.getUrlNetLoc(), self.getUrlBasePath()); + + def getUserAgent(self): + """ + Gets the User-Agent field of the HTTP header, returning empty string + if not present. + """ + return ''; + + def getContentType(self): + """ + Gets the Content-Type field of the HTTP header, parsed into a type + string and a dictionary. + """ + return ('text/html', {}); + + def getContentLength(self): + """ + Gets the content length. + Returns int. + """ + return 0; + + def getBodyIoStream(self): + """ + Returns file object for reading the HTML body. + """ + raise WebServerGlueException('getUrlPath is not implemented'); + + def getBodyIoStreamBinary(self): + """ + Returns file object for reading the binary HTML body. + """ + raise WebServerGlueException('getBodyIoStreamBinary is not implemented'); + + # + # Output stuff. + # + + def _writeHeader(self, sHeaderLine): + """ + Worker function which child classes can override. + """ + sys.stderr.write('_writeHeader: cch=%s "%s..."\n' % (len(sHeaderLine), sHeaderLine[0:10],)) + self.oOutputText.write(sHeaderLine); + return True; + + def flushHeader(self): + """ + Flushes the HTTP header. + """ + if self._fHeaderWrittenOut is False: + for sKey, sValue in self._dHeaderFields.items(): + self._writeHeader('%s: %s\n' % (sKey, sValue,)); + self._fHeaderWrittenOut = True; + self._writeHeader('\n'); # End of header indicator. + return None; + + def setHeaderField(self, sField, sValue): + """ + Sets a header field. + """ + assert self._fHeaderWrittenOut is False; + self._dHeaderFields[sField] = sValue; + return True; + + def setRedirect(self, sLocation, iCode = 302): + """ + Sets up redirection of the page. + Raises an exception if called too late. + """ + if self._fHeaderWrittenOut is True: + raise WebServerGlueException('setRedirect called after the header was written'); + if iCode != 302: + raise WebServerGlueException('Redirection code %d is not supported' % (iCode,)); + + self.setHeaderField('Location', sLocation); + self.setHeaderField('Status', '302 Found'); + return True; + + def setStatus(self, iStatus, sMsg = None): + """ Sets the status code. """ + if not sMsg: + sMsg = self.kdStatusMsgs[iStatus]; + return self.setHeaderField('Status', '%u %s' % (iStatus, sMsg)); + + def setContentType(self, sType): + """ Sets the content type header field. """ + return self.setHeaderField('Content-Type', sType); + + def _writeWorker(self, sChunkOfHtml): + """ + Worker function which child classes can override. + """ + sys.stderr.write('_writeWorker: cch=%s "%s..."\n' % (len(sChunkOfHtml), sChunkOfHtml[0:10],)) + self.oOutputText.write(sChunkOfHtml); + return True; + + def write(self, sChunkOfHtml): + """ + Writes chunk of HTML, making sure the HTTP header is flushed first. + """ + if self._sBodyType is None: + self._sBodyType = 'html'; + elif self._sBodyType != 'html': + raise WebServerGlueException('Cannot use writeParameter when body type is "%s"' % (self._sBodyType, )); + + self._sHtmlBody += sChunkOfHtml; + self._cchCached += len(sChunkOfHtml); + + if self._cchCached > self.kcchMaxCached: + self.flush(); + return True; + + def writeRaw(self, abChunk): + """ + Writes a raw chunk the document. Can be binary or any encoding. + No caching. + """ + if self._sBodyType is None: + self._sBodyType = 'raw'; + elif self._sBodyType != 'raw': + raise WebServerGlueException('Cannot use writeRaw when body type is "%s"' % (self._sBodyType, )); + + self.flushHeader(); + if self._cchCached > 0: + self.flush(); + + sys.stderr.write('writeRaw: cb=%s\n' % (len(abChunk),)) + self.oOutputRaw.write(abChunk); + return True; + + def writeParams(self, dParams): + """ + Writes one or more reply parameters in a form style response. The names + and values in dParams are unencoded, this method takes care of that. + + Note! This automatically changes the content type to + 'application/x-www-form-urlencoded', if the header hasn't been flushed + already. + """ + if self._sBodyType is None: + if not self._fHeaderWrittenOut: + self.setHeaderField('Content-Type', 'application/x-www-form-urlencoded; charset=utf-8'); + elif self._dHeaderFields['Content-Type'] != 'application/x-www-form-urlencoded; charset=utf-8': + raise WebServerGlueException('Cannot use writeParams when content-type is "%s"' % \ + (self._dHeaderFields['Content-Type'],)); + self._sBodyType = 'form'; + + elif self._sBodyType != 'form': + raise WebServerGlueException('Cannot use writeParams when body type is "%s"' % (self._sBodyType, )); + + for sKey in dParams: + sValue = str(dParams[sKey]); + self._dParams[sKey] = sValue; + self._cchCached += len(sKey) + len(sValue); + + if self._cchCached > self.kcchMaxCached: + self.flush(); + + return True; + + def flush(self): + """ + Flush the output. + """ + self.flushHeader(); + + if self._sBodyType == 'form': + sBody = webutils.encodeUrlParams(self._dParams); + self._writeWorker(sBody); + + self._dParams = {}; + self._cchBodyWrittenOut += self._cchCached; + + elif self._sBodyType == 'html': + self._writeWorker(self._sHtmlBody); + + self._sHtmlBody = ''; + self._cchBodyWrittenOut += self._cchCached; + + self._cchCached = 0; + return None; + + # + # Paths. + # + + def pathTmWebUI(self): + """ + Gets the path to the TM 'webui' directory. + """ + return os.path.join(self._sValidationKitDir, 'testmanager', 'webui'); + + # + # Error stuff & Debugging. + # + + def errorLog(self, sError, aXcptInfo, sLogFile): + """ + Writes the error to a log file. + """ + # Easy solution for log file size: Only one report. + try: os.unlink(sLogFile); + except: pass; + + # Try write the log file. + fRc = True; + fSaved = self._fHtmlDebugOutput; + + try: + with open(sLogFile, 'w') as oFile: # pylint: disable=unspecified-encoding + oFile.write(sError + '\n\n'); + if aXcptInfo[0] is not None: + oFile.write(' B a c k t r a c e\n'); + oFile.write('===================\n'); + oFile.write(cgitb.text(aXcptInfo, 5)); + oFile.write('\n\n'); + + oFile.write(' D e b u g I n f o\n'); + oFile.write('=====================\n\n'); + self._fHtmlDebugOutput = False; + self.debugDumpStuff(oFile.write); + except: + fRc = False; + + self._fHtmlDebugOutput = fSaved; + return fRc; + + def errorPage(self, sError, aXcptInfo, sLogFile = None): + """ + Displays a page with an error message. + """ + if sLogFile is not None: + self.errorLog(sError, aXcptInfo, sLogFile); + + # Reset buffering, hoping that nothing was flushed yet. + self._sBodyType = None; + self._sHtmlBody = ''; + self._cchCached = 0; + if not self._fHeaderWrittenOut: + if self._fHtmlDebugOutput: + self.setHeaderField('Content-Type', 'text/html; charset=utf-8'); + else: + self.setHeaderField('Content-Type', 'text/plain; charset=utf-8'); + + # Write the error page. + if self._fHtmlDebugOutput: + self.write('<html><head><title>Test Manage Error</title></head>\n' + + '<body><h1>Test Manager Error:</h1>\n' + + '<p>' + sError + '</p>\n'); + else: + self.write(' Test Manage Error\n' + '===================\n' + '\n' + '' + sError + '\n\n'); + + if aXcptInfo[0] is not None: + if self._fHtmlDebugOutput: + self.write('<h1>Backtrace:</h1>\n'); + self.write(cgitb.html(aXcptInfo, 5)); + else: + self.write('Backtrace\n' + '---------\n' + '\n'); + self.write(cgitb.text(aXcptInfo, 5)); + self.write('\n\n'); + + if self.kfDebugInfoEnabled: + if self._fHtmlDebugOutput: + self.write('<h1>Debug Info:</h1>\n'); + else: + self.write('Debug Info\n' + '----------\n' + '\n'); + self.debugDumpStuff(); + + for fn in self._afnDebugInfo: + try: + fn(self, self._fHtmlDebugOutput); + except Exception as oXcpt: + self.write('\nDebug info callback %s raised exception: %s\n' % (fn, oXcpt)); + + if self._fHtmlDebugOutput: + self.write('</body></html>'); + + self.flush(); + + def debugInfoPage(self, fnWrite = None): + """ + Dumps useful debug info. + """ + if fnWrite is None: + fnWrite = self.write; + + fnWrite('<html><head><title>Test Manage Debug Info</title></head>\n<body>\n'); + self.debugDumpStuff(fnWrite = fnWrite); + fnWrite('</body></html>'); + self.flush(); + + def debugDumpDict(self, sName, dDict, fSorted = True, fnWrite = None): + """ + Dumps dictionary. + """ + if fnWrite is None: + fnWrite = self.write; + + asKeys = list(dDict.keys()); + if fSorted: + asKeys.sort(); + + if self._fHtmlDebugOutput: + fnWrite('<h2>%s</h2>\n' + '<table border="1"><tr><th>name</th><th>value</th></tr>\n' % (sName,)); + for sKey in asKeys: + fnWrite(' <tr><td>' + webutils.escapeElem(sKey) + '</td><td>' \ + + webutils.escapeElem(str(dDict.get(sKey))) \ + + '</td></tr>\n'); + fnWrite('</table>\n'); + else: + for i in range(len(sName) - 1): + fnWrite('%s ' % (sName[i],)); + fnWrite('%s\n\n' % (sName[-1],)); + + fnWrite('%28s Value\n' % ('Name',)); + fnWrite('------------------------------------------------------------------------\n'); + for sKey in asKeys: + fnWrite('%28s: %s\n' % (sKey, dDict.get(sKey),)); + fnWrite('\n'); + + return True; + + def debugDumpList(self, sName, aoStuff, fnWrite = None): + """ + Dumps array. + """ + if fnWrite is None: + fnWrite = self.write; + + if self._fHtmlDebugOutput: + fnWrite('<h2>%s</h2>\n' + '<table border="1"><tr><th>index</th><th>value</th></tr>\n' % (sName,)); + for i, _ in enumerate(aoStuff): + fnWrite(' <tr><td>' + str(i) + '</td><td>' + webutils.escapeElem(str(aoStuff[i])) + '</td></tr>\n'); + fnWrite('</table>\n'); + else: + for ch in sName[:-1]: + fnWrite('%s ' % (ch,)); + fnWrite('%s\n\n' % (sName[-1],)); + + fnWrite('Index Value\n'); + fnWrite('------------------------------------------------------------------------\n'); + for i, oStuff in enumerate(aoStuff): + fnWrite('%5u %s\n' % (i, str(oStuff))); + fnWrite('\n'); + + return True; + + def debugDumpParameters(self, fnWrite): + """ Dumps request parameters. """ + if fnWrite is None: + fnWrite = self.write; + + try: + dParams = self.getParameters(); + return self.debugDumpDict('Parameters', dParams); + except Exception as oXcpt: + if self._fHtmlDebugOutput: + fnWrite('<p>Exception %s while retriving parameters.</p>\n' % (oXcpt,)) + else: + fnWrite('Exception %s while retriving parameters.\n' % (oXcpt,)) + return False; + + def debugDumpEnv(self, fnWrite = None): + """ Dumps os.environ. """ + return self.debugDumpDict('Environment (os.environ)', os.environ, fnWrite = fnWrite); + + def debugDumpArgv(self, fnWrite = None): + """ Dumps sys.argv. """ + return self.debugDumpList('Arguments (sys.argv)', sys.argv, fnWrite = fnWrite); + + def debugDumpPython(self, fnWrite = None): + """ + Dump python info. + """ + dInfo = {}; + dInfo['sys.version'] = sys.version; + dInfo['sys.hexversion'] = sys.hexversion; + dInfo['sys.api_version'] = sys.api_version; + if hasattr(sys, 'subversion'): + dInfo['sys.subversion'] = sys.subversion; # pylint: disable=no-member + dInfo['sys.platform'] = sys.platform; + dInfo['sys.executable'] = sys.executable; + dInfo['sys.copyright'] = sys.copyright; + dInfo['sys.byteorder'] = sys.byteorder; + dInfo['sys.exec_prefix'] = sys.exec_prefix; + dInfo['sys.prefix'] = sys.prefix; + dInfo['sys.path'] = sys.path; + dInfo['sys.builtin_module_names'] = sys.builtin_module_names; + dInfo['sys.flags'] = sys.flags; + + return self.debugDumpDict('Python Info', dInfo, fnWrite = fnWrite); + + + def debugDumpStuff(self, fnWrite = None): + """ + Dumps stuff to the error page and debug info page. + Should be extended by child classes when possible. + """ + self.debugDumpParameters(fnWrite); + self.debugDumpEnv(fnWrite); + self.debugDumpArgv(fnWrite); + self.debugDumpPython(fnWrite); + return True; + + def dprint(self, sMessage): + """ + Prints to debug log (usually apache error log). + """ + if config.g_kfSrvGlueDebug is True: + if config.g_kfSrvGlueDebugTS is False: + self._oDbgFile.write(sMessage); + if not sMessage.endswith('\n'): + self._oDbgFile.write('\n'); + else: + tsNow = utils.timestampMilli(); + tsReq = tsNow - (self.tsStart / 1000000); + iPid = os.getpid(); + for sLine in sMessage.split('\n'): + self._oDbgFile.write('%s/%03u,pid=%04x: %s\n' % (tsNow, tsReq, iPid, sLine,)); + + return True; + + def registerDebugInfoCallback(self, fnDebugInfo): + """ + Registers a debug info method for calling when the error page is shown. + + The fnDebugInfo function takes two parameters. The first is this + object, the second is a boolean indicating html (True) or text (False) + output. The return value is ignored. + """ + if self.kfDebugInfoEnabled: + self._afnDebugInfo.append(fnDebugInfo); + return True; + + def unregisterDebugInfoCallback(self, fnDebugInfo): + """ + Unregisters a debug info method previously registered by + registerDebugInfoCallback. + """ + if self.kfDebugInfoEnabled: + try: self._afnDebugInfo.remove(fnDebugInfo); + except: pass; + return True; + diff --git a/src/VBox/ValidationKit/testmanager/core/webservergluecgi.py b/src/VBox/ValidationKit/testmanager/core/webservergluecgi.py new file mode 100755 index 00000000..e530c7ec --- /dev/null +++ b/src/VBox/ValidationKit/testmanager/core/webservergluecgi.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +# $Id: webservergluecgi.py $ + +""" +Test Manager Core - Web Server Abstraction Base Class. +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Standard python imports. +import cgitb; +import os; +import sys; +import cgi; + +# Validation Kit imports. +from testmanager.core.webservergluebase import WebServerGlueBase; +from testmanager import config; + + +class WebServerGlueCgi(WebServerGlueBase): + """ + CGI glue. + """ + def __init__(self, sValidationKitDir, fHtmlOutput=True): + WebServerGlueBase.__init__(self, sValidationKitDir, fHtmlOutput); + + if config.g_kfSrvGlueCgiTb is True: + cgitb.enable(format=('html' if fHtmlOutput else 'text')); + + def getParameters(self): + return cgi.parse(keep_blank_values=True); + + def getClientAddr(self): + return os.environ.get('REMOTE_ADDR'); + + def getMethod(self): + return os.environ.get('REQUEST_METHOD', 'POST'); + + def getLoginName(self): + return os.environ.get('REMOTE_USER', WebServerGlueBase.ksUnknownUser); + + def getUrlScheme(self): + return 'https' if 'HTTPS' in os.environ else 'http'; + + def getUrlNetLoc(self): + return os.environ['HTTP_HOST']; + + def getUrlPath(self): + return os.environ['REQUEST_URI']; + + def getUserAgent(self): + return os.getenv('HTTP_USER_AGENT', ''); + + def getContentType(self): + return cgi.parse_header(os.environ.get('CONTENT_TYPE', 'text/html')); + + def getContentLength(self): + return int(os.environ.get('CONTENT_LENGTH', 0)); + + def getBodyIoStream(self): + return sys.stdin; + + def getBodyIoStreamBinary(self): + # Python 3: sys.stdin.read() returns a string. To get untranslated + # binary data we use the sys.stdin.buffer object instead. + return getattr(sys.stdin, 'buffer', sys.stdin); + |