diff options
Diffstat (limited to '')
| -rwxr-xr-x | src/VBox/ValidationKit/common/utils.py | 2571 |
1 files changed, 2571 insertions, 0 deletions
diff --git a/src/VBox/ValidationKit/common/utils.py b/src/VBox/ValidationKit/common/utils.py new file mode 100755 index 00000000..e8a50604 --- /dev/null +++ b/src/VBox/ValidationKit/common/utils.py @@ -0,0 +1,2571 @@ +# -*- coding: utf-8 -*- +# $Id: utils.py $ +# pylint: disable=too-many-lines + +""" +Common Utility Functions. +""" + +from __future__ import print_function; + +__copyright__ = \ +""" +Copyright (C) 2012-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see <https://www.gnu.org/licenses>. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + + +# Standard Python imports. +import datetime; +import errno; +import os; +import platform; +import re; +import stat; +import subprocess; +import sys; +import time; +import traceback; +import unittest; + +if sys.platform == 'win32': + import ctypes; + import msvcrt; # pylint: disable=import-error + import win32api; # pylint: disable=import-error + import win32con; # pylint: disable=import-error + import win32console; # pylint: disable=import-error + import win32file; # pylint: disable=import-error + import win32process; # pylint: disable=import-error + import winerror; # pylint: disable=import-error + import pywintypes; # pylint: disable=import-error +else: + import signal; + +# Python 3 hacks: +if sys.version_info[0] >= 3: + unicode = str; # pylint: disable=redefined-builtin,invalid-name + xrange = range; # pylint: disable=redefined-builtin,invalid-name + long = int; # pylint: disable=redefined-builtin,invalid-name + + +# +# Strings. +# + +def toUnicode(sString, encoding = None, errors = 'strict'): + """ + A little like the python 2 unicode() function. + """ + if sys.version_info[0] >= 3: + if isinstance(sString, bytes): + return str(sString, encoding if encoding else 'utf-8', errors); + else: + if not isinstance(sString, unicode): + return unicode(sString, encoding if encoding else 'utf-8', errors); + return sString; + + + +# +# Output. +# + +def printOut(sString): + """ + Outputs a string to standard output, dealing with python 2.x encoding stupidity. + """ + sStreamEncoding = sys.stdout.encoding; + if sStreamEncoding is None: # Files, pipes and such on 2.x. (pylint is confused here) + sStreamEncoding = 'US-ASCII'; # pylint: disable=redefined-variable-type + if sStreamEncoding == 'UTF-8' or not isinstance(sString, unicode): + print(sString); + else: + print(sString.encode(sStreamEncoding, 'backslashreplace').decode(sStreamEncoding)); + +def printErr(sString): + """ + Outputs a string to standard error, dealing with python 2.x encoding stupidity. + """ + sStreamEncoding = sys.stderr.encoding; + if sStreamEncoding is None: # Files, pipes and such on 2.x. (pylint is confused here) + sStreamEncoding = 'US-ASCII'; # pylint: disable=redefined-variable-type + if sStreamEncoding == 'UTF-8' or not isinstance(sString, unicode): + print(sString, file = sys.stderr); + else: + print(sString.encode(sStreamEncoding, 'backslashreplace').decode(sStreamEncoding), file = sys.stderr); + + +# +# Host OS and CPU. +# + +def getHostOs(): + """ + Gets the host OS name (short). + + See the KBUILD_OSES variable in kBuild/header.kmk for possible return values. + """ + sPlatform = platform.system(); + if sPlatform in ('Linux', 'Darwin', 'Solaris', 'FreeBSD', 'NetBSD', 'OpenBSD'): + sPlatform = sPlatform.lower(); + elif sPlatform == 'Windows': + sPlatform = 'win'; + elif sPlatform == 'SunOS': + sPlatform = 'solaris'; + else: + raise Exception('Unsupported platform "%s"' % (sPlatform,)); + return sPlatform; + +g_sHostArch = None; + +def getHostArch(): + """ + Gets the host CPU architecture. + + See the KBUILD_ARCHES variable in kBuild/header.kmk for possible return values. + """ + global g_sHostArch; + if g_sHostArch is None: + sArch = platform.machine(); + if sArch in ('i386', 'i486', 'i586', 'i686', 'i786', 'i886', 'x86'): + sArch = 'x86'; + elif sArch in ('AMD64', 'amd64', 'x86_64'): + sArch = 'amd64'; + elif sArch == 'i86pc': # SunOS + if platform.architecture()[0] == '64bit': + sArch = 'amd64'; + else: + try: + sArch = str(processOutputChecked(['/usr/bin/isainfo', '-n',])); + except: + pass; + sArch = sArch.strip(); + if sArch != 'amd64': + sArch = 'x86'; + else: + raise Exception('Unsupported architecture/machine "%s"' % (sArch,)); + g_sHostArch = sArch; + return g_sHostArch; + + +def getHostOsDotArch(): + """ + Gets the 'os.arch' for the host. + """ + return '%s.%s' % (getHostOs(), getHostArch()); + + +def isValidOs(sOs): + """ + Validates the OS name. + """ + if sOs in ('darwin', 'dos', 'dragonfly', 'freebsd', 'haiku', 'l4', 'linux', 'netbsd', 'nt', 'openbsd', \ + 'os2', 'solaris', 'win', 'os-agnostic'): + return True; + return False; + + +def isValidArch(sArch): + """ + Validates the CPU architecture name. + """ + if sArch in ('x86', 'amd64', 'sparc32', 'sparc64', 's390', 's390x', 'ppc32', 'ppc64', \ + 'mips32', 'mips64', 'ia64', 'hppa32', 'hppa64', 'arm', 'alpha'): + return True; + return False; + +def isValidOsDotArch(sOsDotArch): + """ + Validates the 'os.arch' string. + """ + + asParts = sOsDotArch.split('.'); + if asParts.length() != 2: + return False; + return isValidOs(asParts[0]) \ + and isValidArch(asParts[1]); + +def getHostOsVersion(): + """ + Returns the host OS version. This is platform.release with additional + distro indicator on linux. + """ + sVersion = platform.release(); + sOs = getHostOs(); + if sOs == 'linux': + sDist = ''; + try: + # try /etc/lsb-release first to distinguish between Debian and Ubuntu + with open('/etc/lsb-release') as oFile: # pylint: disable=unspecified-encoding + for sLine in oFile: + oMatch = re.search(r'(?:DISTRIB_DESCRIPTION\s*=)\s*"*(.*)"', sLine); + if oMatch is not None: + sDist = oMatch.group(1).strip(); + except: + pass; + if sDist: + sVersion += ' / ' + sDist; + else: + asFiles = \ + [ + [ '/etc/debian_version', 'Debian v'], + [ '/etc/gentoo-release', '' ], + [ '/etc/oracle-release', '' ], + [ '/etc/redhat-release', '' ], + [ '/etc/SuSE-release', '' ], + ]; + for sFile, sPrefix in asFiles: + if os.path.isfile(sFile): + try: + with open(sFile) as oFile: # pylint: disable=unspecified-encoding + sLine = oFile.readline(); + except: + continue; + sLine = sLine.strip() + if sLine: + sVersion += ' / ' + sPrefix + sLine; + break; + + elif sOs == 'solaris': + sVersion = platform.version(); + if os.path.isfile('/etc/release'): + try: + with open('/etc/release') as oFile: # pylint: disable=unspecified-encoding + sLast = oFile.readlines()[-1]; + sLast = sLast.strip(); + if sLast: + sVersion += ' (' + sLast + ')'; + except: + pass; + + elif sOs == 'darwin': + def getMacVersionName(sVersion): + """ + Figures out the Mac OS X/macOS code name from the numeric version. + """ + aOsVersion = sVersion.split('.') # example: ('10','15','7') + codenames = {"4": "Tiger", + "5": "Leopard", + "6": "Snow Leopard", + "7": "Lion", + "8": "Mountain Lion", + "9": "Mavericks", + "10": "Yosemite", + "11": "El Capitan", + "12": "Sierra", + "13": "High Sierra", + "14": "Mojave", + "15": "Catalina", + "16": "Wrong version", + } + codenames_afterCatalina = {"11": "Big Sur", + "12": "Monterey", + "13": "Ventura", + "14": "Unknown 14", + "15": "Unknown 15"} + + if aOsVersion[0] == '10': + sResult = codenames[aOsVersion[1]] + else: + sResult = codenames_afterCatalina[aOsVersion[0]] + return sResult + + sOsxVersion = platform.mac_ver()[0] + sVersion += ' / OS X ' + sOsxVersion + ' (' + getMacVersionName(sOsxVersion) + ')' + + elif sOs == 'win': + class OSVersionInfoEx(ctypes.Structure): + """ OSVERSIONEX """ + kaFields = [ + ('dwOSVersionInfoSize', ctypes.c_ulong), + ('dwMajorVersion', ctypes.c_ulong), + ('dwMinorVersion', ctypes.c_ulong), + ('dwBuildNumber', ctypes.c_ulong), + ('dwPlatformId', ctypes.c_ulong), + ('szCSDVersion', ctypes.c_wchar*128), + ('wServicePackMajor', ctypes.c_ushort), + ('wServicePackMinor', ctypes.c_ushort), + ('wSuiteMask', ctypes.c_ushort), + ('wProductType', ctypes.c_byte), + ('wReserved', ctypes.c_byte)] + _fields_ = kaFields # pylint: disable=invalid-name + + def __init__(self): + super(OSVersionInfoEx, self).__init__() + self.dwOSVersionInfoSize = ctypes.sizeof(self) + + oOsVersion = OSVersionInfoEx() + rc = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(oOsVersion)) + if rc == 0: + # Python platform.release() is not reliable for newer server releases + if oOsVersion.wProductType != 1: + if oOsVersion.dwMajorVersion == 10 and oOsVersion.dwMinorVersion == 0: + sVersion = '2016Server'; + elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 3: + sVersion = '2012ServerR2'; + elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 2: + sVersion = '2012Server'; + elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 1: + sVersion = '2008ServerR2'; + elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 0: + sVersion = '2008Server'; + elif oOsVersion.dwMajorVersion == 5 and oOsVersion.dwMinorVersion == 2: + sVersion = '2003Server'; + sVersion += ' build ' + str(oOsVersion.dwBuildNumber) + if oOsVersion.wServicePackMajor: + sVersion += ' SP' + str(oOsVersion.wServicePackMajor) + if oOsVersion.wServicePackMinor: + sVersion += '.' + str(oOsVersion.wServicePackMinor) + + return sVersion; + +def getPresentCpuCount(): + """ + Gets the number of CPUs present in the system. + + This differs from multiprocessor.cpu_count() and os.cpu_count() on windows in + that we return the active count rather than the maximum count. If we don't, + we will end up thinking testboxmem1 has 512 CPU threads, which it doesn't and + never will have. + + @todo This is probably not exactly what we get on non-windows... + """ + + if getHostOs() == 'win': + fnGetActiveProcessorCount = getattr(ctypes.windll.kernel32, 'GetActiveProcessorCount', None); + if fnGetActiveProcessorCount: + cCpus = fnGetActiveProcessorCount(ctypes.c_ushort(0xffff)); + if cCpus > 0: + return cCpus; + + import multiprocessing + return multiprocessing.cpu_count(); + + +# +# File system. +# + +def openNoInherit(sFile, sMode = 'r'): + """ + Wrapper around open() that tries it's best to make sure the file isn't + inherited by child processes. + + This is a best effort thing at the moment as it doesn't synchronizes with + child process spawning in any way. Thus it can be subject to races in + multithreaded programs. + """ + + # Python 3.4 and later automatically creates non-inherit handles. See PEP-0446. + uPythonVer = (sys.version_info[0] << 16) | (sys.version_info[1] & 0xffff); + if uPythonVer >= ((3 << 16) | 4): + oFile = open(sFile, sMode); # pylint: disable=consider-using-with,unspecified-encoding + else: + try: + from fcntl import FD_CLOEXEC, F_GETFD, F_SETFD, fcntl; # pylint: disable=import-error + except: + # On windows, we can use the 'N' flag introduced in Visual C++ 7.0 or 7.1 with python 2.x. + if getHostOs() == 'win': + if uPythonVer < (3 << 16): + offComma = sMode.find(','); + if offComma < 0: + return open(sFile, sMode + 'N'); # pylint: disable=consider-using-with,unspecified-encoding + return open(sFile, # pylint: disable=consider-using-with,unspecified-encoding,bad-open-mode + sMode[:offComma] + 'N' + sMode[offComma:]); + + # Just in case. + return open(sFile, sMode); # pylint: disable=consider-using-with,unspecified-encoding + + oFile = open(sFile, sMode); # pylint: disable=consider-using-with,unspecified-encoding + #try: + fcntl(oFile, F_SETFD, fcntl(oFile, F_GETFD) | FD_CLOEXEC); + #except: + # pass; + return oFile; + +def openNoDenyDeleteNoInherit(sFile, sMode = 'r'): + """ + Wrapper around open() that tries it's best to make sure the file isn't + inherited by child processes. + + This is a best effort thing at the moment as it doesn't synchronizes with + child process spawning in any way. Thus it can be subject to races in + multithreaded programs. + """ + + if getHostOs() == 'win': + # Need to use CreateFile directly to open the file so we can feed it FILE_SHARE_DELETE. + # pylint: disable=no-member,c-extension-no-member + fAccess = 0; + fDisposition = win32file.OPEN_EXISTING; + if 'r' in sMode or '+' in sMode: + fAccess |= win32file.GENERIC_READ; + if 'a' in sMode: + fAccess |= win32file.GENERIC_WRITE; + fDisposition = win32file.OPEN_ALWAYS; + elif 'w' in sMode: + fAccess = win32file.GENERIC_WRITE; + if '+' in sMode: + fDisposition = win32file.OPEN_ALWAYS; + fAccess |= win32file.GENERIC_READ; + else: + fDisposition = win32file.CREATE_ALWAYS; + if not fAccess: + fAccess |= win32file.GENERIC_READ; + fSharing = (win32file.FILE_SHARE_READ | win32file.FILE_SHARE_WRITE + | win32file.FILE_SHARE_DELETE); + hFile = win32file.CreateFile(sFile, fAccess, fSharing, None, fDisposition, 0, None); + if 'a' in sMode: + win32file.SetFilePointer(hFile, 0, win32file.FILE_END); + + # Turn the NT handle into a CRT file descriptor. + hDetachedFile = hFile.Detach(); + if fAccess == win32file.GENERIC_READ: + fOpen = os.O_RDONLY; + elif fAccess == win32file.GENERIC_WRITE: + fOpen = os.O_WRONLY; + else: + fOpen = os.O_RDWR; + # pulint: enable=no-member,c-extension-no-member + if 'a' in sMode: + fOpen |= os.O_APPEND; + if 'b' in sMode or 't' in sMode: + fOpen |= os.O_TEXT; # pylint: disable=no-member + fdFile = msvcrt.open_osfhandle(hDetachedFile, fOpen); + + # Tell python to use this handle. + oFile = os.fdopen(fdFile, sMode); + else: + oFile = open(sFile, sMode); # pylint: disable=consider-using-with,unspecified-encoding + + # Python 3.4 and later automatically creates non-inherit handles. See PEP-0446. + uPythonVer = (sys.version_info[0] << 16) | (sys.version_info[1] & 0xffff); + if uPythonVer < ((3 << 16) | 4): + try: + from fcntl import FD_CLOEXEC, F_GETFD, F_SETFD, fcntl; # pylint: disable=import-error + except: + pass; + else: + fcntl(oFile, F_SETFD, fcntl(oFile, F_GETFD) | FD_CLOEXEC); + return oFile; + +def noxcptReadLink(sPath, sXcptRet, sEncoding = 'utf-8'): + """ + No exceptions os.readlink wrapper. + """ + try: + sRet = os.readlink(sPath); # pylint: disable=no-member + except: + return sXcptRet; + if hasattr(sRet, 'decode'): + sRet = sRet.decode(sEncoding, 'ignore'); + return sRet; + +def readFile(sFile, sMode = 'rb'): + """ + Reads the entire file. + """ + with open(sFile, sMode) as oFile: # pylint: disable=unspecified-encoding + sRet = oFile.read(); + return sRet; + +def noxcptReadFile(sFile, sXcptRet, sMode = 'rb', sEncoding = 'utf-8'): + """ + No exceptions common.readFile wrapper. + """ + try: + sRet = readFile(sFile, sMode); + except: + sRet = sXcptRet; + if sEncoding is not None and hasattr(sRet, 'decode'): + sRet = sRet.decode(sEncoding, 'ignore'); + return sRet; + +def noxcptRmDir(sDir, oXcptRet = False): + """ + No exceptions os.rmdir wrapper. + """ + oRet = True; + try: + os.rmdir(sDir); + except: + oRet = oXcptRet; + return oRet; + +def noxcptDeleteFile(sFile, oXcptRet = False): + """ + No exceptions os.remove wrapper. + """ + oRet = True; + try: + os.remove(sFile); + except: + oRet = oXcptRet; + return oRet; + + +def dirEnumerateTree(sDir, fnCallback, fIgnoreExceptions = True): + # type: (string, (string, stat) -> bool) -> bool + """ + Recursively walks a directory tree, calling fnCallback for each. + + fnCallback takes a full path and stat object (can be None). It + returns a boolean value, False stops walking and returns immediately. + + Returns True or False depending on fnCallback. + Returns None fIgnoreExceptions is True and an exception was raised by listdir. + """ + def __worker(sCurDir): + """ Worker for """ + try: + asNames = os.listdir(sCurDir); + except: + if not fIgnoreExceptions: + raise; + return None; + rc = True; + for sName in asNames: + if sName not in [ '.', '..' ]: + sFullName = os.path.join(sCurDir, sName); + try: oStat = os.lstat(sFullName); + except: oStat = None; + if fnCallback(sFullName, oStat) is False: + return False; + if oStat is not None and stat.S_ISDIR(oStat.st_mode): + rc = __worker(sFullName); + if rc is False: + break; + return rc; + + # Ensure unicode path here so listdir also returns unicode on windows. + ## @todo figure out unicode stuff on non-windows. + if sys.platform == 'win32': + sDir = unicode(sDir); + return __worker(sDir); + + + +def formatFileMode(uMode): + # type: (int) -> string + """ + Format a st_mode value 'ls -la' fasion. + Returns string. + """ + if stat.S_ISDIR(uMode): sMode = 'd'; + elif stat.S_ISREG(uMode): sMode = '-'; + elif stat.S_ISLNK(uMode): sMode = 'l'; + elif stat.S_ISFIFO(uMode): sMode = 'p'; + elif stat.S_ISCHR(uMode): sMode = 'c'; + elif stat.S_ISBLK(uMode): sMode = 'b'; + elif stat.S_ISSOCK(uMode): sMode = 's'; + else: sMode = '?'; + ## @todo sticky bits. + sMode += 'r' if uMode & stat.S_IRUSR else '-'; + sMode += 'w' if uMode & stat.S_IWUSR else '-'; + sMode += 'x' if uMode & stat.S_IXUSR else '-'; + sMode += 'r' if uMode & stat.S_IRGRP else '-'; + sMode += 'w' if uMode & stat.S_IWGRP else '-'; + sMode += 'x' if uMode & stat.S_IXGRP else '-'; + sMode += 'r' if uMode & stat.S_IROTH else '-'; + sMode += 'w' if uMode & stat.S_IWOTH else '-'; + sMode += 'x' if uMode & stat.S_IXOTH else '-'; + sMode += ' '; + return sMode; + + +def formatFileStat(oStat): + # type: (stat) -> string + """ + Format a stat result 'ls -la' fasion (numeric IDs). + Returns string. + """ + return '%s %3s %4s %4s %10s %s' \ + % (formatFileMode(oStat.st_mode), oStat.st_nlink, oStat.st_uid, oStat.st_gid, oStat.st_size, + time.strftime('%Y-%m-%d %H:%M', time.localtime(oStat.st_mtime)), ); + +## Good buffer for file operations. +g_cbGoodBufferSize = 256*1024; + +## The original shutil.copyfileobj. +g_fnOriginalShCopyFileObj = None; + +def __myshutilcopyfileobj(fsrc, fdst, length = g_cbGoodBufferSize): + """ shutil.copyfileobj with different length default value (16384 is slow with python 2.7 on windows). """ + return g_fnOriginalShCopyFileObj(fsrc, fdst, length); + +def __installShUtilHacks(shutil): + """ Installs the shutil buffer size hacks. """ + global g_fnOriginalShCopyFileObj; + if g_fnOriginalShCopyFileObj is None: + g_fnOriginalShCopyFileObj = shutil.copyfileobj; + shutil.copyfileobj = __myshutilcopyfileobj; + return True; + + +def copyFileSimple(sFileSrc, sFileDst): + """ + Wrapper around shutil.copyfile that simply copies the data of a regular file. + Raises exception on failure. + Return True for show. + """ + import shutil; + __installShUtilHacks(shutil); + return shutil.copyfile(sFileSrc, sFileDst); + + +def getDiskUsage(sPath): + """ + Get free space of a partition that corresponds to specified sPath in MB. + + Returns partition free space value in MB. + """ + if platform.system() == 'Windows': + oCTypeFreeSpace = ctypes.c_ulonglong(0); + ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(sPath), None, None, + ctypes.pointer(oCTypeFreeSpace)); + cbFreeSpace = oCTypeFreeSpace.value; + else: + oStats = os.statvfs(sPath); # pylint: disable=no-member + cbFreeSpace = long(oStats.f_frsize) * oStats.f_bfree; + + # Convert to MB + cMbFreeSpace = long(cbFreeSpace) / (1024 * 1024); + + return cMbFreeSpace; + + + +# +# SubProcess. +# + +def _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs): + """ + If the "executable" is a python script, insert the python interpreter at + the head of the argument list so that it will work on systems which doesn't + support hash-bang scripts. + """ + + asArgs = dKeywordArgs.get('args'); + if asArgs is None: + asArgs = aPositionalArgs[0]; + + if asArgs[0].endswith('.py'): + if sys.executable: + asArgs.insert(0, sys.executable); + else: + asArgs.insert(0, 'python'); + + # paranoia... + if dKeywordArgs.get('args') is not None: + dKeywordArgs['args'] = asArgs; + else: + aPositionalArgs = (asArgs,) + aPositionalArgs[1:]; + return None; + +def processPopenSafe(*aPositionalArgs, **dKeywordArgs): + """ + Wrapper for subprocess.Popen that's Ctrl-C safe on windows. + """ + if getHostOs() == 'win': + if dKeywordArgs.get('creationflags', 0) == 0: + dKeywordArgs['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP; + return subprocess.Popen(*aPositionalArgs, **dKeywordArgs); # pylint: disable=consider-using-with + +def processStart(*aPositionalArgs, **dKeywordArgs): + """ + Wrapper around subprocess.Popen to deal with its absence in older + python versions. + Returns process object on success which can be worked on. + """ + _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs); + return processPopenSafe(*aPositionalArgs, **dKeywordArgs); + +def processCall(*aPositionalArgs, **dKeywordArgs): + """ + Wrapper around subprocess.call to deal with its absence in older + python versions. + Returns process exit code (see subprocess.poll). + """ + assert dKeywordArgs.get('stdout') is None; + assert dKeywordArgs.get('stderr') is None; + oProcess = processStart(*aPositionalArgs, **dKeywordArgs); + return oProcess.wait(); + +def processOutputChecked(*aPositionalArgs, **dKeywordArgs): + """ + Wrapper around subprocess.check_output to deal with its absense in older + python versions. + + Extra keywords for specifying now output is to be decoded: + sEncoding='utf-8 + fIgnoreEncoding=True/False + """ + sEncoding = dKeywordArgs.get('sEncoding'); + if sEncoding is not None: del dKeywordArgs['sEncoding']; + else: sEncoding = 'utf-8'; + + fIgnoreEncoding = dKeywordArgs.get('fIgnoreEncoding'); + if fIgnoreEncoding is not None: del dKeywordArgs['fIgnoreEncoding']; + else: fIgnoreEncoding = True; + + _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs); + oProcess = processPopenSafe(stdout=subprocess.PIPE, *aPositionalArgs, **dKeywordArgs); + + sOutput, _ = oProcess.communicate(); + iExitCode = oProcess.poll(); + + if iExitCode != 0: + asArgs = dKeywordArgs.get('args'); + if asArgs is None: + asArgs = aPositionalArgs[0]; + print(sOutput); + raise subprocess.CalledProcessError(iExitCode, asArgs); + + if hasattr(sOutput, 'decode'): + sOutput = sOutput.decode(sEncoding, 'ignore' if fIgnoreEncoding else 'strict'); + return sOutput; + +def processOutputUnchecked(*aPositionalArgs, **dKeywordArgs): + """ + Similar to processOutputChecked, but returns status code and both stdout + and stderr results. + + Extra keywords for specifying now output is to be decoded: + sEncoding='utf-8 + fIgnoreEncoding=True/False + """ + sEncoding = dKeywordArgs.get('sEncoding'); + if sEncoding is not None: del dKeywordArgs['sEncoding']; + else: sEncoding = 'utf-8'; + + fIgnoreEncoding = dKeywordArgs.get('fIgnoreEncoding'); + if fIgnoreEncoding is not None: del dKeywordArgs['fIgnoreEncoding']; + else: fIgnoreEncoding = True; + + _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs); + oProcess = processPopenSafe(stdout = subprocess.PIPE, stderr = subprocess.PIPE, *aPositionalArgs, **dKeywordArgs); + + sOutput, sError = oProcess.communicate(); + iExitCode = oProcess.poll(); + + if hasattr(sOutput, 'decode'): + sOutput = sOutput.decode(sEncoding, 'ignore' if fIgnoreEncoding else 'strict'); + if hasattr(sError, 'decode'): + sError = sError.decode(sEncoding, 'ignore' if fIgnoreEncoding else 'strict'); + return (iExitCode, sOutput, sError); + +g_fOldSudo = None; +def _sudoFixArguments(aPositionalArgs, dKeywordArgs, fInitialEnv = True): + """ + Adds 'sudo' (or similar) to the args parameter, whereever it is. + """ + + # Are we root? + fIsRoot = True; + try: + fIsRoot = os.getuid() == 0; # pylint: disable=no-member + except: + pass; + + # If not, prepend sudo (non-interactive, simulate initial login). + if fIsRoot is not True: + asArgs = dKeywordArgs.get('args'); + if asArgs is None: + asArgs = aPositionalArgs[0]; + + # Detect old sudo. + global g_fOldSudo; + if g_fOldSudo is None: + try: + sVersion = str(processOutputChecked(['sudo', '-V'])); + except: + sVersion = '1.7.0'; + sVersion = sVersion.strip().split('\n', 1)[0]; + sVersion = sVersion.replace('Sudo version', '').strip(); + g_fOldSudo = len(sVersion) >= 4 \ + and sVersion[0] == '1' \ + and sVersion[1] == '.' \ + and sVersion[2] <= '6' \ + and sVersion[3] == '.'; + + asArgs.insert(0, 'sudo'); + if not g_fOldSudo: + asArgs.insert(1, '-n'); + if fInitialEnv and not g_fOldSudo: + asArgs.insert(1, '-i'); + + # paranoia... + if dKeywordArgs.get('args') is not None: + dKeywordArgs['args'] = asArgs; + else: + aPositionalArgs = (asArgs,) + aPositionalArgs[1:]; + return None; + + +def sudoProcessStart(*aPositionalArgs, **dKeywordArgs): + """ + sudo (or similar) + subprocess.Popen, + returning the process object on success. + """ + _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs); + _sudoFixArguments(aPositionalArgs, dKeywordArgs); + return processStart(*aPositionalArgs, **dKeywordArgs); + +def sudoProcessCall(*aPositionalArgs, **dKeywordArgs): + """ + sudo (or similar) + subprocess.call + """ + _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs); + _sudoFixArguments(aPositionalArgs, dKeywordArgs); + return processCall(*aPositionalArgs, **dKeywordArgs); + +def sudoProcessOutputChecked(*aPositionalArgs, **dKeywordArgs): + """ + sudo (or similar) + subprocess.check_output. + """ + _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs); + _sudoFixArguments(aPositionalArgs, dKeywordArgs); + return processOutputChecked(*aPositionalArgs, **dKeywordArgs); + +def sudoProcessOutputCheckedNoI(*aPositionalArgs, **dKeywordArgs): + """ + sudo (or similar) + subprocess.check_output, except '-i' isn't used. + """ + _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs); + _sudoFixArguments(aPositionalArgs, dKeywordArgs, False); + return processOutputChecked(*aPositionalArgs, **dKeywordArgs); + +def sudoProcessPopen(*aPositionalArgs, **dKeywordArgs): + """ + sudo (or similar) + processPopenSafe. + """ + _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs); + _sudoFixArguments(aPositionalArgs, dKeywordArgs); + return processPopenSafe(*aPositionalArgs, **dKeywordArgs); + + +def whichProgram(sName, sPath = None): + """ + Works similar to the 'which' utility on unix. + + Returns path to the given program if found. + Returns None if not found. + """ + sHost = getHostOs(); + sSep = ';' if sHost in [ 'win', 'os2' ] else ':'; + + if sPath is None: + if sHost == 'win': + sPath = os.environ.get('Path', None); + else: + sPath = os.environ.get('PATH', None); + if sPath is None: + return None; + + for sDir in sPath.split(sSep): + if sDir.strip() != '': + sTest = os.path.abspath(os.path.join(sDir, sName)); + else: + sTest = os.path.abspath(sName); + if os.path.exists(sTest): + return sTest; + + return None; + +# +# Generic process stuff. +# + +def processInterrupt(uPid): + """ + Sends a SIGINT or equivalent to interrupt the specified process. + Returns True on success, False on failure. + + On Windows hosts this may not work unless the process happens to be a + process group leader. + """ + if sys.platform == 'win32': + try: + win32console.GenerateConsoleCtrlEvent(win32con.CTRL_BREAK_EVENT, # pylint: disable=no-member,c-extension-no-member + uPid); + fRc = True; + except: + fRc = False; + else: + try: + os.kill(uPid, signal.SIGINT); + fRc = True; + except: + fRc = False; + return fRc; + +def sendUserSignal1(uPid): + """ + Sends a SIGUSR1 or equivalent to nudge the process into shutting down + (VBoxSVC) or something. + Returns True on success, False on failure or if not supported (win). + + On Windows hosts this may not work unless the process happens to be a + process group leader. + """ + if sys.platform == 'win32': + fRc = False; + else: + try: + os.kill(uPid, signal.SIGUSR1); # pylint: disable=no-member + fRc = True; + except: + fRc = False; + return fRc; + +def processTerminate(uPid): + """ + Terminates the process in a nice manner (SIGTERM or equivalent). + Returns True on success, False on failure. + """ + fRc = False; + if sys.platform == 'win32': + try: + hProcess = win32api.OpenProcess(win32con.PROCESS_TERMINATE, # pylint: disable=no-member,c-extension-no-member + False, uPid); + except: + pass; + else: + try: + win32process.TerminateProcess(hProcess, # pylint: disable=no-member,c-extension-no-member + 0x40010004); # DBG_TERMINATE_PROCESS + fRc = True; + except: + pass; + hProcess.Close(); #win32api.CloseHandle(hProcess) + else: + try: + os.kill(uPid, signal.SIGTERM); + fRc = True; + except: + pass; + return fRc; + +def processKill(uPid): + """ + Terminates the process with extreme prejudice (SIGKILL). + Returns True on success, False on failure. + """ + if sys.platform == 'win32': + fRc = processTerminate(uPid); + else: + try: + os.kill(uPid, signal.SIGKILL); # pylint: disable=no-member + fRc = True; + except: + fRc = False; + return fRc; + +def processKillWithNameCheck(uPid, sName): + """ + Like processKill(), but checks if the process name matches before killing + it. This is intended for killing using potentially stale pid values. + + Returns True on success, False on failure. + """ + + if processCheckPidAndName(uPid, sName) is not True: + return False; + return processKill(uPid); + + +def processExists(uPid): + """ + Checks if the specified process exits. + This will only work if we can signal/open the process. + + Returns True if it positively exists, False otherwise. + """ + sHostOs = getHostOs(); + if sHostOs == 'win': + fRc = False; + # We try open the process for waiting since this is generally only forbidden in a very few cases. + try: + hProcess = win32api.OpenProcess(win32con.SYNCHRONIZE, # pylint: disable=no-member,c-extension-no-member + False, uPid); + except pywintypes.error as oXcpt: # pylint: disable=no-member + if oXcpt.winerror == winerror.ERROR_ACCESS_DENIED: + fRc = True; + except: + pass; + else: + hProcess.Close(); + fRc = True; + else: + fRc = False; + try: + os.kill(uPid, 0); + fRc = True; + except OSError as oXcpt: + if oXcpt.errno == errno.EPERM: + fRc = True; + except: + pass; + return fRc; + +def processCheckPidAndName(uPid, sName): + """ + Checks if a process PID and NAME matches. + """ + fRc = processExists(uPid); + if fRc is not True: + return False; + + if sys.platform == 'win32': + try: + from win32com.client import GetObject; # pylint: disable=import-error + oWmi = GetObject('winmgmts:'); + aoProcesses = oWmi.InstancesOf('Win32_Process'); + for oProcess in aoProcesses: + if long(oProcess.Properties_("ProcessId").Value) == uPid: + sCurName = oProcess.Properties_("Name").Value; + #reporter.log2('uPid=%s sName=%s sCurName=%s' % (uPid, sName, sCurName)); + sName = sName.lower(); + sCurName = sCurName.lower(); + if os.path.basename(sName) == sName: + sCurName = os.path.basename(sCurName); + + if sCurName == sName \ + or sCurName + '.exe' == sName \ + or sCurName == sName + '.exe': + fRc = True; + break; + except: + #reporter.logXcpt('uPid=%s sName=%s' % (uPid, sName)); + pass; + else: + if sys.platform in ('linux2', 'linux', 'linux3', 'linux4', 'linux5', 'linux6'): + asPsCmd = ['/bin/ps', '-p', '%u' % (uPid,), '-o', 'fname=']; + elif sys.platform in ('sunos5',): + asPsCmd = ['/usr/bin/ps', '-p', '%u' % (uPid,), '-o', 'fname=']; + elif sys.platform in ('darwin',): + asPsCmd = ['/bin/ps', '-p', '%u' % (uPid,), '-o', 'ucomm=']; + else: + asPsCmd = None; + + if asPsCmd is not None: + try: + oPs = subprocess.Popen(asPsCmd, stdout=subprocess.PIPE); # pylint: disable=consider-using-with + sCurName = oPs.communicate()[0]; + iExitCode = oPs.wait(); + except: + #reporter.logXcpt(); + return False; + + # ps fails with non-zero exit code if the pid wasn't found. + if iExitCode != 0: + return False; + if sCurName is None: + return False; + sCurName = sCurName.strip(); + if not sCurName: + return False; + + if os.path.basename(sName) == sName: + sCurName = os.path.basename(sCurName); + elif os.path.basename(sCurName) == sCurName: + sName = os.path.basename(sName); + + if sCurName != sName: + return False; + + fRc = True; + return fRc; + +def processGetInfo(uPid, fSudo = False): + """ + Tries to acquire state information of the given process. + + Returns a string with the information on success or None on failure or + if the host is not supported. + + Note that the format of the information is host system dependent and will + likely differ much between different hosts. + """ + fRc = processExists(uPid); + if fRc is not True: + return None; + + sHostOs = getHostOs(); + if sHostOs in [ 'linux',]: + sGdb = '/usr/bin/gdb'; + if not os.path.isfile(sGdb): sGdb = '/usr/local/bin/gdb'; + if not os.path.isfile(sGdb): sGdb = 'gdb'; + aasCmd = [ + [ sGdb, '-batch', + '-ex', 'set pagination off', + '-ex', 'thread apply all bt', + '-ex', 'info proc mapping', + '-ex', 'info sharedlibrary', + '-p', '%u' % (uPid,), ], + ]; + elif sHostOs == 'darwin': + # LLDB doesn't work in batch mode when attaching to a process, at least + # with macOS Sierra (10.12). GDB might not be installed. Use the sample + # tool instead with a 1 second duration and 1000ms sampling interval to + # get one stack trace. For the process mappings use vmmap. + aasCmd = [ + [ '/usr/bin/sample', '-mayDie', '%u' % (uPid,), '1', '1000', ], + [ '/usr/bin/vmmap', '%u' % (uPid,), ], + ]; + elif sHostOs == 'solaris': + aasCmd = [ + [ '/usr/bin/pstack', '%u' % (uPid,), ], + [ '/usr/bin/pmap', '%u' % (uPid,), ], + ]; + else: + aasCmd = []; + + sInfo = ''; + for asCmd in aasCmd: + try: + if fSudo: + sThisInfo = sudoProcessOutputChecked(asCmd); + else: + sThisInfo = processOutputChecked(asCmd); + if sThisInfo is not None: + sInfo += sThisInfo; + except: + pass; + if not sInfo: + sInfo = None; + + return sInfo; + + +class ProcessInfo(object): + """Process info.""" + def __init__(self, iPid): + self.iPid = iPid; + self.iParentPid = None; + self.sImage = None; + self.sName = None; + self.asArgs = None; + self.sCwd = None; + self.iGid = None; + self.iUid = None; + self.iProcGroup = None; + self.iSessionId = None; + + def loadAll(self): + """Load all the info.""" + sOs = getHostOs(); + if sOs == 'linux': + sProc = '/proc/%s/' % (self.iPid,); + if self.sImage is None: self.sImage = noxcptReadLink(sProc + 'exe', None); + if self.sImage is None: + self.sImage = noxcptReadFile(sProc + 'comm', None); + if self.sImage: self.sImage = self.sImage.strip(); + if self.sCwd is None: self.sCwd = noxcptReadLink(sProc + 'cwd', None); + if self.asArgs is None: self.asArgs = noxcptReadFile(sProc + 'cmdline', '').split('\x00'); + #elif sOs == 'solaris': - doesn't work for root processes, suid proces, and other stuff. + # sProc = '/proc/%s/' % (self.iPid,); + # if self.sImage is None: self.sImage = noxcptReadLink(sProc + 'path/a.out', None); + # if self.sCwd is None: self.sCwd = noxcptReadLink(sProc + 'path/cwd', None); + else: + pass; + if self.sName is None and self.sImage is not None: + self.sName = self.sImage; + + def windowsGrabProcessInfo(self, oProcess): + """Windows specific loadAll.""" + try: self.sName = oProcess.Properties_("Name").Value; + except: pass; + try: self.sImage = oProcess.Properties_("ExecutablePath").Value; + except: pass; + try: self.asArgs = [oProcess.Properties_("CommandLine").Value]; ## @todo split it. + except: pass; + try: self.iParentPid = oProcess.Properties_("ParentProcessId").Value; + except: pass; + try: self.iSessionId = oProcess.Properties_("SessionId").Value; + except: pass; + if self.sName is None and self.sImage is not None: + self.sName = self.sImage; + + def getBaseImageName(self): + """ + Gets the base image name if available, use the process name if not available. + Returns image/process base name or None. + """ + sRet = self.sImage if self.sName is None else self.sName; + if sRet is None: + self.loadAll(); + sRet = self.sImage if self.sName is None else self.sName; + if sRet is None: + if not self.asArgs: + return None; + sRet = self.asArgs[0]; + if not sRet: + return None; + return os.path.basename(sRet); + + def getBaseImageNameNoExeSuff(self): + """ + Same as getBaseImageName, except any '.exe' or similar suffix is stripped. + """ + sRet = self.getBaseImageName(); + if sRet is not None and len(sRet) > 4 and sRet[-4] == '.': + if (sRet[-4:]).lower() in [ '.exe', '.com', '.msc', '.vbs', '.cmd', '.bat' ]: + sRet = sRet[:-4]; + return sRet; + + +def processListAll(): + """ + Return a list of ProcessInfo objects for all the processes in the system + that the current user can see. + """ + asProcesses = []; + + sOs = getHostOs(); + if sOs == 'win': + from win32com.client import GetObject; # pylint: disable=import-error + oWmi = GetObject('winmgmts:'); + aoProcesses = oWmi.InstancesOf('Win32_Process'); + for oProcess in aoProcesses: + try: + iPid = int(oProcess.Properties_("ProcessId").Value); + except: + continue; + oMyInfo = ProcessInfo(iPid); + oMyInfo.windowsGrabProcessInfo(oProcess); + asProcesses.append(oMyInfo); + return asProcesses; + + if sOs in [ 'linux', ]: # Not solaris, ps gets more info than /proc/. + try: + asDirs = os.listdir('/proc'); + except: + asDirs = []; + for sDir in asDirs: + if sDir.isdigit(): + asProcesses.append(ProcessInfo(int(sDir),)); + return asProcesses; + + # + # The other OSes parses the output from the 'ps' utility. + # + asPsCmd = [ + '/bin/ps', # 0 + '-A', # 1 + '-o', 'pid=', # 2,3 + '-o', 'ppid=', # 4,5 + '-o', 'pgid=', # 6,7 + '-o', 'sid=', # 8,9 + '-o', 'uid=', # 10,11 + '-o', 'gid=', # 12,13 + '-o', 'comm=' # 14,15 + ]; + + if sOs == 'darwin': + assert asPsCmd[9] == 'sid='; + asPsCmd[9] = 'sess='; + elif sOs == 'solaris': + asPsCmd[0] = '/usr/bin/ps'; + + try: + sRaw = processOutputChecked(asPsCmd); + except: + return asProcesses; + + for sLine in sRaw.split('\n'): + sLine = sLine.lstrip(); + if len(sLine) < 7 or not sLine[0].isdigit(): + continue; + + iField = 0; + off = 0; + aoFields = [None, None, None, None, None, None, None]; + while iField < 7: + # Eat whitespace. + while off < len(sLine) and (sLine[off] == ' ' or sLine[off] == '\t'): + off += 1; + + # Final field / EOL. + if iField == 6: + aoFields[6] = sLine[off:]; + break; + if off >= len(sLine): + break; + + # Generic field parsing. + offStart = off; + off += 1; + while off < len(sLine) and sLine[off] != ' ' and sLine[off] != '\t': + off += 1; + try: + if iField != 3: + aoFields[iField] = int(sLine[offStart:off]); + else: + aoFields[iField] = long(sLine[offStart:off], 16); # sess is a hex address. + except: + pass; + iField += 1; + + if aoFields[0] is not None: + oMyInfo = ProcessInfo(aoFields[0]); + oMyInfo.iParentPid = aoFields[1]; + oMyInfo.iProcGroup = aoFields[2]; + oMyInfo.iSessionId = aoFields[3]; + oMyInfo.iUid = aoFields[4]; + oMyInfo.iGid = aoFields[5]; + oMyInfo.sName = aoFields[6]; + asProcesses.append(oMyInfo); + + return asProcesses; + + +def processCollectCrashInfo(uPid, fnLog, fnCrashFile): + """ + Looks for information regarding the demise of the given process. + """ + sOs = getHostOs(); + if sOs == 'darwin': + # + # On darwin we look for crash and diagnostic reports. + # + asLogDirs = [ + u'/Library/Logs/DiagnosticReports/', + u'/Library/Logs/CrashReporter/', + u'~/Library/Logs/DiagnosticReports/', + u'~/Library/Logs/CrashReporter/', + ]; + for sDir in asLogDirs: + sDir = os.path.expanduser(sDir); + if not os.path.isdir(sDir): + continue; + try: + asDirEntries = os.listdir(sDir); + except: + continue; + for sEntry in asDirEntries: + # Only interested in .crash files. + _, sSuff = os.path.splitext(sEntry); + if sSuff != '.crash': + continue; + + # The pid can be found at the end of the first line. + sFull = os.path.join(sDir, sEntry); + try: + with open(sFull, 'r') as oFile: # pylint: disable=unspecified-encoding + sFirstLine = oFile.readline(); + except: + continue; + if len(sFirstLine) <= 4 or sFirstLine[-2] != ']': + continue; + offPid = len(sFirstLine) - 3; + while offPid > 1 and sFirstLine[offPid - 1].isdigit(): + offPid -= 1; + try: uReportPid = int(sFirstLine[offPid:-2]); + except: continue; + + # Does the pid we found match? + if uReportPid == uPid: + fnLog('Found crash report for %u: %s' % (uPid, sFull,)); + fnCrashFile(sFull, False); + elif sOs == 'win': + # + # Getting WER reports would be great, however we have trouble match the + # PID to those as they seems not to mention it in the brief reports. + # Instead we'll just look for crash dumps in C:\CrashDumps (our custom + # location - see the windows readme for the testbox script) and what + # the MSDN article lists for now. + # + # It's been observed on Windows server 2012 that the dump files takes + # the form: <processimage>.<decimal-pid>.dmp + # + asDmpDirs = [ + u'%SystemDrive%/CrashDumps/', # Testboxes. + u'%LOCALAPPDATA%/CrashDumps/', # MSDN example. + u'%WINDIR%/ServiceProfiles/LocalServices/', # Local and network service. + u'%WINDIR%/ServiceProfiles/NetworkSerices/', + u'%WINDIR%/ServiceProfiles/', + u'%WINDIR%/System32/Config/SystemProfile/', # System services. + ]; + sMatchSuffix = '.%u.dmp' % (uPid,); + + for sDir in asDmpDirs: + sDir = os.path.expandvars(sDir); + if not os.path.isdir(sDir): + continue; + try: + asDirEntries = os.listdir(sDir); + except: + continue; + for sEntry in asDirEntries: + if sEntry.endswith(sMatchSuffix): + sFull = os.path.join(sDir, sEntry); + fnLog('Found crash dump for %u: %s' % (uPid, sFull,)); + fnCrashFile(sFull, True); + elif sOs == 'solaris': + asDmpDirs = []; + try: + sScratchPath = os.environ.get('TESTBOX_PATH_SCRATCH', None); + asDmpDirs.extend([ sScratchPath ]); + except: + pass; + # Some other useful locations as fallback. + asDmpDirs.extend([ + u'/var/cores/', + u'/var/core/', + ]); + # + # Solaris by default creates a core file in the directory of the crashing process with the name 'core'. + # + # As we need to distinguish the core files correlating to their PIDs and have a persistent storage location, + # the host needs to be tweaked via: + # + # ```coreadm -g /path/to/cores/core.%f.%p``` + # + sMatchSuffix = '.%u.core' % (uPid,); + for sDir in asDmpDirs: + sDir = os.path.expandvars(sDir); + if not os.path.isdir(sDir): + continue; + try: + asDirEntries = os.listdir(sDir); + except: + continue; + for sEntry in asDirEntries: + fnLog('Entry: %s' % (os.path.join(sDir, sEntry))); + if sEntry.endswith(sMatchSuffix): + sFull = os.path.join(sDir, sEntry); + fnLog('Found crash dump for %u: %s' % (uPid, sFull,)); + fnCrashFile(sFull, True); + else: + pass; ## TODO + return None; + + +# +# Time. +# + +# +# The following test case shows how time.time() only have ~ms resolution +# on Windows (tested W10) and why it therefore makes sense to try use +# performance counters. +# +# Note! We cannot use time.clock() as the timestamp must be portable across +# processes. See timeout testcase problem on win hosts (no logs). +# Also, time.clock() was axed in python 3.8 (https://bugs.python.org/issue31803). +# +#import sys; +#import time; +#from common import utils; +# +#atSeries = []; +#for i in xrange(1,160): +# if i == 159: time.sleep(10); +# atSeries.append((utils.timestampNano(), long(time.clock() * 1000000000), long(time.time() * 1000000000))); +# +#tPrev = atSeries[0] +#for tCur in atSeries: +# print 't1=%+22u, %u' % (tCur[0], tCur[0] - tPrev[0]); +# print 't2=%+22u, %u' % (tCur[1], tCur[1] - tPrev[1]); +# print 't3=%+22u, %u' % (tCur[2], tCur[2] - tPrev[2]); +# print ''; +# tPrev = tCur +# +#print 't1=%u' % (atSeries[-1][0] - atSeries[0][0]); +#print 't2=%u' % (atSeries[-1][1] - atSeries[0][1]); +#print 't3=%u' % (atSeries[-1][2] - atSeries[0][2]); + +g_fWinUseWinPerfCounter = sys.platform == 'win32'; +g_fpWinPerfCounterFreq = None; +g_oFuncwinQueryPerformanceCounter = None; + +def _winInitPerfCounter(): + """ Initializes the use of performance counters. """ + global g_fWinUseWinPerfCounter, g_fpWinPerfCounterFreq, g_oFuncwinQueryPerformanceCounter + + uFrequency = ctypes.c_ulonglong(0); + if ctypes.windll.kernel32.QueryPerformanceFrequency(ctypes.byref(uFrequency)): + if uFrequency.value >= 1000: + #print 'uFrequency = %s' % (uFrequency,); + #print 'type(uFrequency) = %s' % (type(uFrequency),); + g_fpWinPerfCounterFreq = float(uFrequency.value); + + # Check that querying the counter works too. + global g_oFuncwinQueryPerformanceCounter + g_oFuncwinQueryPerformanceCounter = ctypes.windll.kernel32.QueryPerformanceCounter; + uCurValue = ctypes.c_ulonglong(0); + if g_oFuncwinQueryPerformanceCounter(ctypes.byref(uCurValue)): + if uCurValue.value > 0: + return True; + g_fWinUseWinPerfCounter = False; + return False; + +def _winFloatTime(): + """ Gets floating point time on windows. """ + if g_fpWinPerfCounterFreq is not None or _winInitPerfCounter(): + uCurValue = ctypes.c_ulonglong(0); + if g_oFuncwinQueryPerformanceCounter(ctypes.byref(uCurValue)): + return float(uCurValue.value) / g_fpWinPerfCounterFreq; + return time.time(); + +def timestampNano(): + """ + Gets a nanosecond timestamp. + """ + if g_fWinUseWinPerfCounter is True: + return long(_winFloatTime() * 1000000000); + return long(time.time() * 1000000000); + +def timestampMilli(): + """ + Gets a millisecond timestamp. + """ + if g_fWinUseWinPerfCounter is True: + return long(_winFloatTime() * 1000); + return long(time.time() * 1000); + +def timestampSecond(): + """ + Gets a second timestamp. + """ + if g_fWinUseWinPerfCounter is True: + return long(_winFloatTime()); + return long(time.time()); + +def secondsSinceUnixEpoch(): + """ + Returns unix time, floating point second count since 1970-01-01T00:00:00Z + """ + ## ASSUMES This returns unix epoch time on all systems we care about... + return time.time(); + +def getTimePrefix(): + """ + Returns a timestamp prefix, typically used for logging. UTC. + """ + try: + oNow = datetime.datetime.utcnow(); + sTs = '%02u:%02u:%02u.%06u' % (oNow.hour, oNow.minute, oNow.second, oNow.microsecond); + except: + sTs = 'getTimePrefix-exception'; + return sTs; + +def getTimePrefixAndIsoTimestamp(): + """ + Returns current UTC as log prefix and iso timestamp. + """ + try: + oNow = datetime.datetime.utcnow(); + sTsPrf = '%02u:%02u:%02u.%06u' % (oNow.hour, oNow.minute, oNow.second, oNow.microsecond); + sTsIso = formatIsoTimestamp(oNow); + except: + sTsPrf = sTsIso = 'getTimePrefix-exception'; + return (sTsPrf, sTsIso); + +class UtcTzInfo(datetime.tzinfo): + """UTC TZ Info Class""" + def utcoffset(self, _): + return datetime.timedelta(0); + def tzname(self, _): + return "UTC"; + def dst(self, _): + return datetime.timedelta(0); + +class GenTzInfo(datetime.tzinfo): + """Generic TZ Info Class""" + def __init__(self, offInMin): + datetime.tzinfo.__init__(self); + self.offInMin = offInMin; + def utcoffset(self, _): + return datetime.timedelta(minutes = self.offInMin); + def tzname(self, _): + if self.offInMin >= 0: + return "+%02d%02d" % (self.offInMin // 60, self.offInMin % 60); + return "-%02d%02d" % (-self.offInMin // 60, -self.offInMin % 60); + def dst(self, _): + return datetime.timedelta(0); + +def formatIsoTimestamp(oNow): + """Formats the datetime object as an ISO timestamp.""" + assert oNow.tzinfo is None or isinstance(oNow.tzinfo, UtcTzInfo); + sTs = '%s.%09uZ' % (oNow.strftime('%Y-%m-%dT%H:%M:%S'), oNow.microsecond * 1000); + return sTs; + +def getIsoTimestamp(): + """Returns the current UTC timestamp as a string.""" + return formatIsoTimestamp(datetime.datetime.utcnow()); + +def formatShortIsoTimestamp(oNow): + """Formats the datetime object as an ISO timestamp, but w/o microseconds.""" + assert oNow.tzinfo is None or isinstance(oNow.tzinfo, UtcTzInfo); + return oNow.strftime('%Y-%m-%dT%H:%M:%SZ'); + +def getShortIsoTimestamp(): + """Returns the current UTC timestamp as a string, but w/o microseconds.""" + return formatShortIsoTimestamp(datetime.datetime.utcnow()); + +def convertDateTimeToZulu(oDateTime): + """ Converts oDateTime to zulu time if it has timezone info. """ + if oDateTime.tzinfo is not None: + oDateTime = oDateTime.astimezone(UtcTzInfo()); + else: + oDateTime = oDateTime.replace(tzinfo = UtcTzInfo()); + return oDateTime; + +def parseIsoTimestamp(sTs): + """ + Parses a typical ISO timestamp, returing a datetime object, reasonably + forgiving, but will throw weird indexing/conversion errors if the input + is malformed. + """ + # YYYY-MM-DD + iYear = int(sTs[0:4]); + assert(sTs[4] == '-'); + iMonth = int(sTs[5:7]); + assert(sTs[7] == '-'); + iDay = int(sTs[8:10]); + + # Skip separator + sTime = sTs[10:]; + while sTime[0] in 'Tt \t\n\r': + sTime = sTime[1:]; + + # HH:MM[:SS] + iHour = int(sTime[0:2]); + assert(sTime[2] == ':'); + iMin = int(sTime[3:5]); + if sTime[5] == ':': + iSec = int(sTime[6:8]); + + # Fraction? + offTime = 8; + iMicroseconds = 0; + if offTime < len(sTime) and sTime[offTime] in '.,': + offTime += 1; + cchFraction = 0; + while offTime + cchFraction < len(sTime) and sTime[offTime + cchFraction] in '0123456789': + cchFraction += 1; + if cchFraction > 0: + iMicroseconds = int(sTime[offTime : (offTime + cchFraction)]); + offTime += cchFraction; + while cchFraction < 6: + iMicroseconds *= 10; + cchFraction += 1; + while cchFraction > 6: + iMicroseconds = iMicroseconds // 10; + cchFraction -= 1; + + else: + iSec = 0; + iMicroseconds = 0; + offTime = 5; + + # Naive? + if offTime >= len(sTime): + return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds); + + # Zulu? + if offTime >= len(sTime) or sTime[offTime] in 'Zz': + return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds, tzinfo = UtcTzInfo()); + + # Some kind of offset afterwards, and strptime is useless. sigh. + if sTime[offTime] in '+-': + chSign = sTime[offTime]; + offTime += 1; + cMinTz = int(sTime[offTime : (offTime + 2)]) * 60; + offTime += 2; + if offTime < len(sTime) and sTime[offTime] in ':': + offTime += 1; + if offTime + 2 <= len(sTime): + cMinTz += int(sTime[offTime : (offTime + 2)]); + offTime += 2; + assert offTime == len(sTime); + if chSign == '-': + cMinTz = -cMinTz; + return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds, tzinfo = GenTzInfo(cMinTz)); + assert False, sTs; + return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds); + +def normalizeIsoTimestampToZulu(sTs): + """ + Takes a iso timestamp string and normalizes it (basically parseIsoTimestamp + + convertDateTimeToZulu + formatIsoTimestamp). + Returns ISO tiemstamp string. + """ + return formatIsoTimestamp(convertDateTimeToZulu(parseIsoTimestamp(sTs))); + +def getLocalHourOfWeek(): + """ Local hour of week (0 based). """ + oNow = datetime.datetime.now(); + return (oNow.isoweekday() - 1) * 24 + oNow.hour; + + +def formatIntervalSeconds(cSeconds): + """ Format a seconds interval into a nice 01h 00m 22s string """ + # Two simple special cases. + if cSeconds < 60: + return '%ss' % (cSeconds,); + if cSeconds < 3600: + cMins = cSeconds // 60; + cSecs = cSeconds % 60; + if cSecs == 0: + return '%sm' % (cMins,); + return '%sm %ss' % (cMins, cSecs,); + + # Generic and a bit slower. + cDays = cSeconds // 86400; + cSeconds %= 86400; + cHours = cSeconds // 3600; + cSeconds %= 3600; + cMins = cSeconds // 60; + cSecs = cSeconds % 60; + sRet = ''; + if cDays > 0: + sRet = '%sd ' % (cDays,); + if cHours > 0: + sRet += '%sh ' % (cHours,); + if cMins > 0: + sRet += '%sm ' % (cMins,); + if cSecs > 0: + sRet += '%ss ' % (cSecs,); + assert sRet; assert sRet[-1] == ' '; + return sRet[:-1]; + +def formatIntervalSeconds2(oSeconds): + """ + Flexible input version of formatIntervalSeconds for use in WUI forms where + data is usually already string form. + """ + if isinstance(oSeconds, (int, long)): + return formatIntervalSeconds(oSeconds); + if not isString(oSeconds): + try: + lSeconds = long(oSeconds); + except: + pass; + else: + if lSeconds >= 0: + return formatIntervalSeconds2(lSeconds); + return oSeconds; + +def parseIntervalSeconds(sString): + """ + Reverse of formatIntervalSeconds. + + Returns (cSeconds, sError), where sError is None on success. + """ + + # We might given non-strings, just return them without any fuss. + if not isString(sString): + if isinstance(sString, (int, long)) or sString is None: + return (sString, None); + ## @todo time/date objects? + return (int(sString), None); + + # Strip it and make sure it's not empty. + sString = sString.strip(); + if not sString: + return (0, 'Empty interval string.'); + + # + # Split up the input into a list of 'valueN, unitN, ...'. + # + # Don't want to spend too much time trying to make re.split do exactly what + # I need here, so please forgive the extra pass I'm making here. + # + asRawParts = re.split(r'\s*([0-9]+)\s*([^0-9,;]*)[\s,;]*', sString); + asParts = []; + for sPart in asRawParts: + sPart = sPart.strip(); + if sPart: + asParts.append(sPart); + if not asParts: + return (0, 'Empty interval string or something?'); + + # + # Process them one or two at the time. + # + cSeconds = 0; + asErrors = []; + i = 0; + while i < len(asParts): + sNumber = asParts[i]; + i += 1; + if sNumber.isdigit(): + iNumber = int(sNumber); + + sUnit = 's'; + if i < len(asParts) and not asParts[i].isdigit(): + sUnit = asParts[i]; + i += 1; + + sUnitLower = sUnit.lower(); + if sUnitLower in [ 's', 'se', 'sec', 'second', 'seconds' ]: + pass; + elif sUnitLower in [ 'm', 'mi', 'min', 'minute', 'minutes' ]: + iNumber *= 60; + elif sUnitLower in [ 'h', 'ho', 'hou', 'hour', 'hours' ]: + iNumber *= 3600; + elif sUnitLower in [ 'd', 'da', 'day', 'days' ]: + iNumber *= 86400; + elif sUnitLower in [ 'w', 'week', 'weeks' ]: + iNumber *= 7 * 86400; + else: + asErrors.append('Unknown unit "%s".' % (sUnit,)); + cSeconds += iNumber; + else: + asErrors.append('Bad number "%s".' % (sNumber,)); + return (cSeconds, None if not asErrors else ' '.join(asErrors)); + +def formatIntervalHours(cHours): + """ Format a hours interval into a nice 1w 2d 1h string. """ + # Simple special cases. + if cHours < 24: + return '%sh' % (cHours,); + + # Generic and a bit slower. + cWeeks = cHours / (7 * 24); + cHours %= 7 * 24; + cDays = cHours / 24; + cHours %= 24; + sRet = ''; + if cWeeks > 0: + sRet = '%sw ' % (cWeeks,); + if cDays > 0: + sRet = '%sd ' % (cDays,); + if cHours > 0: + sRet += '%sh ' % (cHours,); + assert sRet; assert sRet[-1] == ' '; + return sRet[:-1]; + +def parseIntervalHours(sString): + """ + Reverse of formatIntervalHours. + + Returns (cHours, sError), where sError is None on success. + """ + + # We might given non-strings, just return them without any fuss. + if not isString(sString): + if isinstance(sString, (int, long)) or sString is None: + return (sString, None); + ## @todo time/date objects? + return (int(sString), None); + + # Strip it and make sure it's not empty. + sString = sString.strip(); + if not sString: + return (0, 'Empty interval string.'); + + # + # Split up the input into a list of 'valueN, unitN, ...'. + # + # Don't want to spend too much time trying to make re.split do exactly what + # I need here, so please forgive the extra pass I'm making here. + # + asRawParts = re.split(r'\s*([0-9]+)\s*([^0-9,;]*)[\s,;]*', sString); + asParts = []; + for sPart in asRawParts: + sPart = sPart.strip(); + if sPart: + asParts.append(sPart); + if not asParts: + return (0, 'Empty interval string or something?'); + + # + # Process them one or two at the time. + # + cHours = 0; + asErrors = []; + i = 0; + while i < len(asParts): + sNumber = asParts[i]; + i += 1; + if sNumber.isdigit(): + iNumber = int(sNumber); + + sUnit = 'h'; + if i < len(asParts) and not asParts[i].isdigit(): + sUnit = asParts[i]; + i += 1; + + sUnitLower = sUnit.lower(); + if sUnitLower in [ 'h', 'ho', 'hou', 'hour', 'hours' ]: + pass; + elif sUnitLower in [ 'd', 'da', 'day', 'days' ]: + iNumber *= 24; + elif sUnitLower in [ 'w', 'week', 'weeks' ]: + iNumber *= 7 * 24; + else: + asErrors.append('Unknown unit "%s".' % (sUnit,)); + cHours += iNumber; + else: + asErrors.append('Bad number "%s".' % (sNumber,)); + return (cHours, None if not asErrors else ' '.join(asErrors)); + + +# +# Introspection. +# + +def getCallerName(oFrame=None, iFrame=2): + """ + Returns the name of the caller's caller. + """ + if oFrame is None: + try: + raise Exception(); + except: + oFrame = sys.exc_info()[2].tb_frame.f_back; + while iFrame > 1: + if oFrame is not None: + oFrame = oFrame.f_back; + iFrame = iFrame - 1; + if oFrame is not None: + sName = '%s:%u' % (oFrame.f_code.co_name, oFrame.f_lineno); + return sName; + return "unknown"; + + +def getXcptInfo(cFrames = 1): + """ + Gets text detailing the exception. (Good for logging.) + Returns list of info strings. + """ + + # + # Try get exception info. + # + try: + oType, oValue, oTraceback = sys.exc_info(); + except: + oType = oValue = oTraceback = None; + if oType is not None: + + # + # Try format the info + # + asRet = []; + try: + try: + asRet = asRet + traceback.format_exception_only(oType, oValue); + asTraceBack = traceback.format_tb(oTraceback); + if cFrames is not None and cFrames <= 1: + asRet.append(asTraceBack[-1]); + else: + asRet.append('Traceback:') + for iFrame in range(min(cFrames, len(asTraceBack))): + asRet.append(asTraceBack[-iFrame - 1]); + asRet.append('Stack:') + asRet = asRet + traceback.format_stack(oTraceback.tb_frame.f_back, cFrames); + except: + asRet.append('internal-error: Hit exception #2! %s' % (traceback.format_exc(),)); + + if not asRet: + asRet.append('No exception info...'); + except: + asRet.append('internal-error: Hit exception! %s' % (traceback.format_exc(),)); + else: + asRet = ['Couldn\'t find exception traceback.']; + return asRet; + + +def getObjectTypeName(oObject): + """ + Get the type name of the given object. + """ + if oObject is None: + return 'None'; + + # Get the type object. + try: + oType = type(oObject); + except: + return 'type-throws-exception'; + + # Python 2.x only: Handle old-style object wrappers. + if sys.version_info[0] < 3: + try: + from types import InstanceType; # pylint: disable=no-name-in-module + if oType == InstanceType: + oType = oObject.__class__; + except: + pass; + + # Get the name. + try: + return oType.__name__; + except: + return '__type__-throws-exception'; + + +def chmodPlusX(sFile): + """ + Makes the specified file or directory executable. + Returns success indicator, no exceptions. + + Note! Symbolic links are followed and the target will be changed. + """ + try: + oStat = os.stat(sFile); + except: + return False; + try: + os.chmod(sFile, oStat.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH); + except: + return False; + return True; + + +# +# TestSuite stuff. +# + +def isRunningFromCheckout(cScriptDepth = 1): + """ + Checks if we're running from the SVN checkout or not. + """ + + try: + sFile = __file__; + cScriptDepth = 1; + except: + sFile = sys.argv[0]; + + sDir = os.path.abspath(sFile); + while cScriptDepth >= 0: + sDir = os.path.dirname(sDir); + if os.path.exists(os.path.join(sDir, 'Makefile.kmk')) \ + or os.path.exists(os.path.join(sDir, 'Makefile.kup')): + return True; + cScriptDepth -= 1; + + return False; + + +# +# Bourne shell argument fun. +# + + +def argsSplit(sCmdLine): + """ + Given a bourne shell command line invocation, split it up into arguments + assuming IFS is space. + Returns None on syntax error. + """ + ## @todo bourne shell argument parsing! + return sCmdLine.split(' '); + +def argsGetFirst(sCmdLine): + """ + Given a bourne shell command line invocation, get return the first argument + assuming IFS is space. + Returns None on invalid syntax, otherwise the parsed and unescaped argv[0] string. + """ + asArgs = argsSplit(sCmdLine); + if not asArgs: + return None; + + return asArgs[0]; + +# +# String helpers. +# + +def stricmp(sFirst, sSecond): + """ + Compares to strings in an case insensitive fashion. + + Python doesn't seem to have any way of doing the correctly, so this is just + an approximation using lower. + """ + if sFirst == sSecond: + return 0; + sLower1 = sFirst.lower(); + sLower2 = sSecond.lower(); + if sLower1 == sLower2: + return 0; + if sLower1 < sLower2: + return -1; + return 1; + + +def versionCompare(sVer1, sVer2): + """ + Compares to version strings in a fashion similar to RTStrVersionCompare. + """ + + ## @todo implement me!! + + if sVer1 == sVer2: + return 0; + if sVer1 < sVer2: + return -1; + return 1; + + +def formatNumber(lNum, sThousandSep = ' '): + """ + Formats a decimal number with pretty separators. + """ + sNum = str(lNum); + sRet = sNum[-3:]; + off = len(sNum) - 3; + while off > 0: + off -= 3; + sRet = sNum[(off if off >= 0 else 0):(off + 3)] + sThousandSep + sRet; + return sRet; + + +def formatNumberNbsp(lNum): + """ + Formats a decimal number with pretty separators. + """ + sRet = formatNumber(lNum); + return unicode(sRet).replace(' ', u'\u00a0'); + + +def isString(oString): + """ + Checks if the object is a string object, hiding difference between python 2 and 3. + + Returns True if it's a string of some kind. + Returns False if not. + """ + if sys.version_info[0] >= 3: + return isinstance(oString, str); + return isinstance(oString, basestring); # pylint: disable=undefined-variable + + +def hasNonAsciiCharacters(sText): + """ + Returns True is specified string has non-ASCII characters, False if ASCII only. + """ + if isString(sText): + for ch in sText: + if ord(ch) >= 128: + return True; + else: + # Probably byte array or some such thing. + for ch in sText: + if ch >= 128 or ch < 0: + return True; + return False; + + +# +# Unpacking. +# + +def unpackZipFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None): + # type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string] + """ + Worker for unpackFile that deals with ZIP files, same function signature. + """ + import zipfile + if fnError is None: + fnError = fnLog; + + fnLog('Unzipping "%s" to "%s"...' % (sArchive, sDstDir)); + + # Open it. + try: oZipFile = zipfile.ZipFile(sArchive, 'r'); # pylint: disable=consider-using-with + except Exception as oXcpt: + fnError('Error opening "%s" for unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt,)); + return None; + + # Extract all members. + asMembers = []; + try: + for sMember in oZipFile.namelist(): + if fnFilter is None or fnFilter(sMember) is not False: + if sMember.endswith('/'): + os.makedirs(os.path.join(sDstDir, sMember.replace('/', os.path.sep)), 0x1fd); # octal: 0775 (python 3/2) + else: + oZipFile.extract(sMember, sDstDir); + asMembers.append(os.path.join(sDstDir, sMember.replace('/', os.path.sep))); + except Exception as oXcpt: + fnError('Error unpacking "%s" into "%s": %s' % (sArchive, sDstDir, oXcpt)); + asMembers = None; + + # close it. + try: oZipFile.close(); + except Exception as oXcpt: + fnError('Error closing "%s" after unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt)); + asMembers = None; + + return asMembers; + + +## Set if we've replaced tarfile.copyfileobj with __mytarfilecopyfileobj already. +g_fTarCopyFileObjOverriddend = False; + +def __mytarfilecopyfileobj(src, dst, length = None, exception = OSError, bufsize = None): + """ tarfile.copyfileobj with different buffer size (16384 is slow on windows). """ + _ = bufsize; + if length is None: + __myshutilcopyfileobj(src, dst, g_cbGoodBufferSize); + elif length > 0: + cFull, cbRemainder = divmod(length, g_cbGoodBufferSize); + for _ in xrange(cFull): + abBuffer = src.read(g_cbGoodBufferSize); + dst.write(abBuffer); + if len(abBuffer) != g_cbGoodBufferSize: + raise exception('unexpected end of source file'); + if cbRemainder > 0: + abBuffer = src.read(cbRemainder); + dst.write(abBuffer); + if len(abBuffer) != cbRemainder: + raise exception('unexpected end of source file'); + + +def unpackTarFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None): + # type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string] + """ + Worker for unpackFile that deals with tarballs, same function signature. + """ + import shutil; + import tarfile; + if fnError is None: + fnError = fnLog; + + fnLog('Untarring "%s" to "%s"...' % (sArchive, sDstDir)); + + # + # Default buffer sizes of 16384 bytes is causing too many syscalls on Windows. + # 60%+ speedup for python 2.7 and 50%+ speedup for python 3.5, both on windows with PDBs. + # 20%+ speedup for python 2.7 and 15%+ speedup for python 3.5, both on windows skipping PDBs. + # + if True is True: # pylint: disable=comparison-with-itself + __installShUtilHacks(shutil); + global g_fTarCopyFileObjOverriddend; + if g_fTarCopyFileObjOverriddend is False: + g_fTarCopyFileObjOverriddend = True; + #if sys.hexversion < 0x03060000: + tarfile.copyfileobj = __mytarfilecopyfileobj; + + # + # Open it. + # + # Note! We not using 'r:*' because we cannot allow seeking compressed files! + # That's how we got a 13 min unpack time for VBoxAll on windows (hardlinked pdb). + # + try: + if sys.hexversion >= 0x03060000: + oTarFile = tarfile.open(sArchive, 'r|*', # pylint: disable=consider-using-with + bufsize = g_cbGoodBufferSize, copybufsize = g_cbGoodBufferSize); + else: + oTarFile = tarfile.open(sArchive, 'r|*', bufsize = g_cbGoodBufferSize); # pylint: disable=consider-using-with + except Exception as oXcpt: + fnError('Error opening "%s" for unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt,)); + return None; + + # Extract all members. + asMembers = []; + try: + for oTarInfo in oTarFile: + try: + if fnFilter is None or fnFilter(oTarInfo.name) is not False: + if oTarInfo.islnk(): + # Links are trouble, especially on Windows. We must avoid the falling that will end up seeking + # in the compressed tar stream. So, fall back on shutil.copy2 instead. + sLinkFile = os.path.join(sDstDir, oTarInfo.name.rstrip('/').replace('/', os.path.sep)); + sLinkTarget = os.path.join(sDstDir, oTarInfo.linkname.rstrip('/').replace('/', os.path.sep)); + sParentDir = os.path.dirname(sLinkFile); + try: os.unlink(sLinkFile); + except: pass; + if sParentDir and not os.path.exists(sParentDir): + os.makedirs(sParentDir); + try: os.link(sLinkTarget, sLinkFile); + except: shutil.copy2(sLinkTarget, sLinkFile); + else: + if oTarInfo.isdir(): + # Just make sure the user (we) got full access to dirs. Don't bother getting it 100% right. + oTarInfo.mode |= 0x1c0; # (octal: 0700) + oTarFile.extract(oTarInfo, sDstDir); + asMembers.append(os.path.join(sDstDir, oTarInfo.name.replace('/', os.path.sep))); + except Exception as oXcpt: + fnError('Error unpacking "%s" member "%s" into "%s": %s' % (sArchive, oTarInfo.name, sDstDir, oXcpt)); + for sAttr in [ 'name', 'linkname', 'type', 'mode', 'size', 'mtime', 'uid', 'uname', 'gid', 'gname' ]: + fnError('Info: %8s=%s' % (sAttr, getattr(oTarInfo, sAttr),)); + for sFn in [ 'isdir', 'isfile', 'islnk', 'issym' ]: + fnError('Info: %8s=%s' % (sFn, getattr(oTarInfo, sFn)(),)); + asMembers = None; + break; + except Exception as oXcpt: + fnError('Error unpacking "%s" into "%s": %s' % (sArchive, sDstDir, oXcpt)); + asMembers = None; + + # + # Finally, close it. + # + try: oTarFile.close(); + except Exception as oXcpt: + fnError('Error closing "%s" after unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt)); + asMembers = None; + + return asMembers; + + +def unpackFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None): + # type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string] + """ + Unpacks the given file if it has a know archive extension, otherwise do + nothing. + + fnLog & fnError both take a string parameter. + + fnFilter takes a member name (string) and returns True if it's included + and False if excluded. + + Returns list of the extracted files (full path) on success. + Returns empty list if not a supported archive format. + Returns None on failure. Raises no exceptions. + """ + sBaseNameLower = os.path.basename(sArchive).lower(); + + # + # Zip file? + # + if sBaseNameLower.endswith('.zip'): + return unpackZipFile(sArchive, sDstDir, fnLog, fnError, fnFilter); + + # + # Tarball? + # + if sBaseNameLower.endswith('.tar') \ + or sBaseNameLower.endswith('.tar.gz') \ + or sBaseNameLower.endswith('.tgz') \ + or sBaseNameLower.endswith('.tar.bz2'): + return unpackTarFile(sArchive, sDstDir, fnLog, fnError, fnFilter); + + # + # Cannot classify it from the name, so just return that to the caller. + # + fnLog('Not unpacking "%s".' % (sArchive,)); + return []; + + +# +# Misc. +# +def areBytesEqual(oLeft, oRight): + """ + Compares two byte arrays, strings or whatnot. + + returns true / false accordingly. + """ + + # If both are None, consider them equal (bogus?): + if oLeft is None and oRight is None: + return True; + + # If just one is None, they can't match: + if oLeft is None or oRight is None: + return False; + + # If both have the same type, use the compare operator of the class: + if type(oLeft) is type(oRight): + #print('same type: %s' % (oLeft == oRight,)); + return oLeft == oRight; + + # On the offchance that they're both strings, but of different types. + if isString(oLeft) and isString(oRight): + #print('string compare: %s' % (oLeft == oRight,)); + return oLeft == oRight; + + # + # See if byte/buffer stuff that can be compared directory. If not convert + # strings to bytes. + # + # Note! For 2.x, we must convert both sides to the buffer type or the + # comparison may fail despite it working okay in test cases. + # + if sys.version_info[0] >= 3: + if isinstance(oLeft, (bytearray, memoryview, bytes)) and isinstance(oRight, (bytearray, memoryview, bytes)): # pylint: disable=undefined-variable + return oLeft == oRight; + + if isString(oLeft): + try: oLeft = bytes(oLeft, 'utf-8'); + except: pass; + if isString(oRight): + try: oRight = bytes(oRight, 'utf-8'); + except: pass; + else: + if isinstance(oLeft, (bytearray, buffer)) and isinstance(oRight, (bytearray, buffer)): # pylint: disable=undefined-variable + if isinstance(oLeft, bytearray): + oLeft = buffer(oLeft); # pylint: disable=redefined-variable-type,undefined-variable + else: + oRight = buffer(oRight); # pylint: disable=redefined-variable-type,undefined-variable + #print('buf/byte #1 compare: %s (%s vs %s)' % (oLeft == oRight, type(oLeft), type(oRight),)); + return oLeft == oRight; + + if isString(oLeft): + try: oLeft = bytearray(oLeft, 'utf-8'); # pylint: disable=redefined-variable-type + except: pass; + if isString(oRight): + try: oRight = bytearray(oRight, 'utf-8'); # pylint: disable=redefined-variable-type + except: pass; + + # Check if we now have the same type for both: + if type(oLeft) is type(oRight): + #print('same type now: %s' % (oLeft == oRight,)); + return oLeft == oRight; + + # Check if we now have buffer/memoryview vs bytes/bytesarray again. + if sys.version_info[0] >= 3: + if isinstance(oLeft, (bytearray, memoryview, bytes)) and isinstance(oRight, (bytearray, memoryview, bytes)): # pylint: disable=undefined-variable + return oLeft == oRight; + else: + if isinstance(oLeft, (bytearray, buffer)) and isinstance(oRight, (bytearray, buffer)): # pylint: disable=undefined-variable + if isinstance(oLeft, bytearray): + oLeft = buffer(oLeft); # pylint: disable=redefined-variable-type,undefined-variable + else: + oRight = buffer(oRight); # pylint: disable=redefined-variable-type,undefined-variable + #print('buf/byte #2 compare: %s (%s vs %s)' % (oLeft == oRight, type(oLeft), type(oRight),)); + return oLeft == oRight; + + # Do item by item comparison: + if len(oLeft) != len(oRight): + #print('different length: %s vs %s' % (len(oLeft), len(oRight))); + return False; + i = len(oLeft); + while i > 0: + i = i - 1; + + iElmLeft = oLeft[i]; + if not isinstance(iElmLeft, int) and not isinstance(iElmLeft, long): + iElmLeft = ord(iElmLeft); + + iElmRight = oRight[i]; + if not isinstance(iElmRight, int) and not isinstance(iElmRight, long): + iElmRight = ord(iElmRight); + + if iElmLeft != iElmRight: + #print('element %d differs: %x %x' % (i, iElmLeft, iElmRight,)); + return False; + return True; + + +def calcCrc32OfFile(sFile): + """ + Simple helper for calculating the CRC32 of a file. + + Throws stuff if the file cannot be opened or read successfully. + """ + import zlib; + + uCrc32 = 0; + with open(sFile, 'rb') as oFile: # pylint: disable=unspecified-encoding + while True: + oBuf = oFile.read(1024 * 1024); + if not oBuf: + break + uCrc32 = zlib.crc32(oBuf, uCrc32); + + return uCrc32 % 2**32; + + +# +# Unit testing. +# + +# pylint: disable=missing-docstring +# pylint: disable=undefined-variable +class BuildCategoryDataTestCase(unittest.TestCase): + def testIntervalSeconds(self): + self.assertEqual(parseIntervalSeconds(formatIntervalSeconds(3600)), (3600, None)); + self.assertEqual(parseIntervalSeconds(formatIntervalSeconds(1209438593)), (1209438593, None)); + self.assertEqual(parseIntervalSeconds('123'), (123, None)); + self.assertEqual(parseIntervalSeconds(123), (123, None)); + self.assertEqual(parseIntervalSeconds(99999999999), (99999999999, None)); + self.assertEqual(parseIntervalSeconds(''), (0, 'Empty interval string.')); + self.assertEqual(parseIntervalSeconds('1X2'), (3, 'Unknown unit "X".')); + self.assertEqual(parseIntervalSeconds('1 Y3'), (4, 'Unknown unit "Y".')); + self.assertEqual(parseIntervalSeconds('1 Z 4'), (5, 'Unknown unit "Z".')); + self.assertEqual(parseIntervalSeconds('1 hour 2m 5second'), (3725, None)); + self.assertEqual(parseIntervalSeconds('1 hour,2m ; 5second'), (3725, None)); + + def testZuluNormalization(self): + self.assertEqual(normalizeIsoTimestampToZulu('2011-01-02T03:34:25.000000000Z'), '2011-01-02T03:34:25.000000000Z'); + self.assertEqual(normalizeIsoTimestampToZulu('2011-01-02T03:04:25-0030'), '2011-01-02T03:34:25.000000000Z'); + self.assertEqual(normalizeIsoTimestampToZulu('2011-01-02T03:04:25+0030'), '2011-01-02T02:34:25.000000000Z'); + self.assertEqual(normalizeIsoTimestampToZulu('2020-03-20T20:47:39,832312863+01:00'), '2020-03-20T19:47:39.832312000Z'); + self.assertEqual(normalizeIsoTimestampToZulu('2020-03-20T20:47:39,832312863-02:00'), '2020-03-20T22:47:39.832312000Z'); + + def testHasNonAsciiChars(self): + self.assertEqual(hasNonAsciiCharacters(''), False); + self.assertEqual(hasNonAsciiCharacters('asdfgebASDFKJ@#$)(!@#UNASDFKHB*&$%&)@#(!)@(#!(#$&*#$&%*Y@#$IQWN---00;'), False); + self.assertEqual(hasNonAsciiCharacters('\x80 '), True); + self.assertEqual(hasNonAsciiCharacters('\x79 '), False); + self.assertEqual(hasNonAsciiCharacters(u'12039889y!@#$%^&*()0-0asjdkfhoiuyweasdfASDFnvV'), False); + self.assertEqual(hasNonAsciiCharacters(u'\u0079'), False); + self.assertEqual(hasNonAsciiCharacters(u'\u0080'), True); + self.assertEqual(hasNonAsciiCharacters(u'\u0081 \u0100'), True); + self.assertEqual(hasNonAsciiCharacters(b'\x20\x20\x20'), False); + self.assertEqual(hasNonAsciiCharacters(b'\x20\x81\x20'), True); + + def testAreBytesEqual(self): + self.assertEqual(areBytesEqual(None, None), True); + self.assertEqual(areBytesEqual(None, ''), False); + self.assertEqual(areBytesEqual('', ''), True); + self.assertEqual(areBytesEqual('1', '1'), True); + self.assertEqual(areBytesEqual('12345', '1234'), False); + self.assertEqual(areBytesEqual('1234', '1234'), True); + self.assertEqual(areBytesEqual('1234', b'1234'), True); + self.assertEqual(areBytesEqual(b'1234', b'1234'), True); + self.assertEqual(areBytesEqual(b'1234', '1234'), True); + self.assertEqual(areBytesEqual(b'1234', bytearray([0x31,0x32,0x33,0x34])), True); + self.assertEqual(areBytesEqual('1234', bytearray([0x31,0x32,0x33,0x34])), True); + self.assertEqual(areBytesEqual(u'1234', bytearray([0x31,0x32,0x33,0x34])), True); + self.assertEqual(areBytesEqual(bytearray([0x31,0x32,0x33,0x34]), bytearray([0x31,0x32,0x33,0x34])), True); + self.assertEqual(areBytesEqual(bytearray([0x31,0x32,0x33,0x34]), '1224'), False); + self.assertEqual(areBytesEqual(bytearray([0x31,0x32,0x33,0x34]), bytearray([0x31,0x32,0x32,0x34])), False); + if sys.version_info[0] >= 3: + pass; + else: + self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), + bytearray([0x31,0x32,0x33,0x34])), True); + self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), + bytearray([0x99,0x32,0x32,0x34])), False); + self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), + buffer(bytearray([0x31,0x32,0x33,0x34,0x34]), 0, 4)), True); + self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), + buffer(bytearray([0x99,0x32,0x33,0x34,0x34]), 0, 4)), False); + self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), b'1234'), True); + self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), '1234'), True); + self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), u'1234'), True); + +if __name__ == '__main__': + unittest.main(); + # not reached. |