testboxscript_real.py revision cf22150eaeeb72431bf1cf65c309a431454fb22b
# -*- coding: utf-8 -*-
# $Id$
"""
TestBox Script - main().
"""
__copyright__ = \
"""
Copyright (C) 2012-2014 Oracle Corporation
This file is part of VirtualBox Open Source Edition (OSE), as
available from http://www.virtualbox.org. This file is free software;
General Public License (GPL) as published by the Free Software
Foundation, in version 2 as it comes in the "COPYING" file of the
VirtualBox OSE distribution. VirtualBox OSE is distributed in the
hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
The contents of this file may alternatively be used under the terms
of the Common Development and Distribution License Version 1.0
(CDDL) only, as it comes in the "COPYING.CDDL" file of the
VirtualBox OSE distribution, in which case the provisions of the
CDDL are applicable instead of those of the GPL.
You may elect to license modified versions of this file under the
terms and conditions of either the GPL or the CDDL or both.
"""
__version__ = "$Revision$"
# Standard python imports.
import math
import multiprocessing
import os
from optparse import OptionParser
import platform
import random
import shutil
import sys
import tempfile
import time
import uuid
# Only the main script needs to modify the path.
try: __file__
# Validation Kit imports.
import testboxcommons;
from testboxcommons import TestBoxException;
from testboxcommand import TestBoxCommand;
from testboxconnection import TestBoxConnection;
class TestBoxScriptException(Exception):
""" For raising exceptions during TestBoxScript.__init__. """
pass;
class TestBoxScript(object):
"""
Implementation of the test box script.
Communicate with test manager and perform offered actions.
"""
## @name Class Constants.
# @{
# Scratch space round value (MB).
# Memory size round value (MB).
# A NULL UUID in string form.
ksNullUuid = '00000000-0000-0000-0000-000000000000';
# The minimum dispatch loop delay.
kcSecMinDelay = 12;
# The maximum dispatch loop delay (inclusive).
kcSecMaxDelay = 24;
# The minimum sign-on delay.
kcSecMinSignOnDelay = 30;
# The maximum sign-on delay (inclusive).
kcSecMaxSignOnDelay = 60;
# Keys for config params
VALUE = 'value'
## @}
"""
Initialize internals
"""
self._sTestBoxHelper = None;
# Signed-on state
self._idTestBox = None;
# Command processor.
#
# many many GBs for some test scenarios and /tmp can be backed by swap
# /tmp often is, this would break host panic / triple-fault detection.
#
# We need *lots* of space, so avoid /tmp as it may be a memory
# file system backed by the swap file, or worse.
else:
sSubDir = 'testbox';
try:
except:
pass;
for sDir in [self._oOptions.sScratchRoot, self._sScratchSpill, self._sScratchScripts, self._sScratchState]:
# We count consecutive reinitScratch failures and will reboot the
# testbox after a while in the hope that it will correct the issue.
#
# Mount builds and test resources if requested.
#
self.mountShares();
#
# Sign-on parameters: Packed into list of records of format:
# { <Parameter ID>: { <Current value>, <Check function> } }
#
self._ddSignOnParams = \
{
constants.tbreq.SIGNON_PARAM_CPU_REVISION: { self.VALUE: self._getHostCpuRevision(), self.FN: None },
constants.tbreq.SIGNON_PARAM_HAS_NESTED_PAGING:{ self.VALUE: self._hasHostNestedPaging(), self.FN: None },
constants.tbreq.SIGNON_PARAM_HAS_64_BIT_GUEST: { self.VALUE: self._can64BitGuest(), self.FN: None },
constants.tbreq.SIGNON_PARAM_PYTHON_VERSION: { self.VALUE: self._getPythonHexVersion(), self.FN: None },
constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE: { self.VALUE: None, self.FN: self._getFreeScratchSpace },
}
#
# The System UUID is the primary identification of the machine, so
# refuse to cooperate if it's NULL.
#
raise TestBoxScriptException('Couldn\'t determine the System UUID, please use --system-uuid to specify it.');
#
# Export environment variables, clearing any we don't know yet.
#
else: # Starts with '=', bad user.
os.environ['TESTBOX_HAS_NESTED_PAGING'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_NESTED_PAGING);
os.environ['TESTBOX_SCRATCH_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE);
# Currently omitting any kBuild tools.
def mountShares(self):
"""
Mounts the shares.
Raises exception on failure.
"""
self._mountShare(self._oOptions.sBuildsPath, self._oOptions.sBuildsServerType, self._oOptions.sBuildsServerName,
self._mountShare(self._oOptions.sTestRsrcPath, self._oOptions.sTestRsrcServerType, self._oOptions.sTestRsrcServerName,
return True;
"""
Mounts the specified share if needed.
Raises exception on failure.
"""
# Only mount if the type is specified.
if sType is None:
return True;
# Test if already mounted.
return True;
#
# Platform specific mount code.
#
if sType == 'cifs':
utils.processOutputChecked(['/sbin/mount_smbfs', '-o', 'automounted,nostreams,soft,noowners,noatime,rdonly',
'-f', '0555', '-d', '0555',
sMountPoint]);
else:
elif sHostOs == 'linux':
if sType == 'cifs':
'-o',
'user=' + sUser
+ ',password=' + sPassword
+ ',sec=ntlmv2'
+ ',nounix,file_mode=0555,dir_mode=0555,soft,ro',
sMountPoint]);
elif sType == 'nfs':
'-o', 'soft,ro',
sMountPoint]);
else:
elif sHostOs == 'solaris':
if sType == 'cifs':
## @todo This stuff doesn't work on wei01-x4600b.de.oracle.com running 11.1. FIXME!
oPasswdFile.flush();
'-o',
'user=' + sUser
+ ',fileperms=0555,dirperms=0555,noxattr,ro',
stdin = oPasswdFile);
oPasswdFile.close();
elif sType == 'nfs':
'-o', 'noxattr,ro',
sMountPoint]);
else:
elif sHostOs == 'win':
if sType != 'cifs':
raise TestBoxScriptException('Only CIFS mounts are supported on Windows.');
'/USER:' + sUser,]);
else:
#
# Re-test.
#
raise TestBoxException('Failed to mount %s (%s[%s]) at %s: %s not found'
return True;
## @name Signon property releated methods.
# @{
"""
Invokes TestBoxHelper to obtain information hard to access from python.
"""
if self._sTestBoxHelper is None:
if not utils.isRunningFromCheckout():
# See VBoxTestBoxScript.zip for layout.
'TestBoxHelper');
else: # Only for in-tree testing, so don't bother be too accurate right now.
'TestBoxHelper');
"""
Invokes TestBoxHelper to obtain information hard to access from python.
"""
if sValue == 'true':
return True;
if sValue == 'false':
return False;
return fDunnoValue;
def _isUuidGood(sUuid):
"""
Checks if the UUID looks good.
There are systems with really bad UUIDs, for instance
"03000200-0400-0500-0006-000700080009".
"""
return False;
return False;
return True;
def _getHostSystemUuid(self):
"""
Get the system UUID string from the System, return null-uuid if
unable to get retrieve it.
"""
#
# Try get at the firmware UUID.
#
# NOTE: This requires to have kernel option enabled:
# Firmware Drivers -> Export DMI identification via sysfs to userspace
try:
except:
pass;
## @todo consider dmidecoder? What about EFI systems?
# Windows: WMI
try:
except:
pass;
try:
except:
pass;
# Solaris: The smbios util.
try:
except:
pass;
return sUuid;
#
# Try add the MAC address.
# uuid.getnode may provide it, or it may return a random number...
#
return sUuid;
def _getHostCpuVendor(self):
"""
Get the CPUID vendor string on intel HW.
"""
def _getHostCpuName(self):
"""
Get the CPU name/description string.
"""
def _getHostCpuRevision(self):
"""
"""
def _hasHostHwVirt(self):
"""
Check if the host supports AMD-V or VT-x
"""
def _hasHostNestedPaging(self):
"""
Check if the host supports nested paging.
"""
if not self._hasHostHwVirt():
return False;
def _can64BitGuest(self):
"""
Check if the we (VBox) can run 64-bit guests.
"""
if not self._hasHostHwVirt():
return False;
def _hasHostIoMmu(self):
"""
Check if the host has an I/O MMU of the VT-d kind.
"""
if not self._hasHostHwVirt():
return False;
## @todo Any way to figure this one out on any host OS?
def _getHostReport(self):
"""
Generate a report about the host hardware and software.
"""
def _getHostMemSize(self):
"""
Gets the amount of physical memory on the host (and accessible to the
OS, i.e. don't report stuff over 4GB if Windows doesn't wanna use it).
Unit: MiB.
"""
# Round it.
return cMbMemory;
def _getFreeScratchSpace(self):
"""
Get free space on the volume where scratch directory is located and
return it in bytes rounded down to nearest 64MB
(currently works on Linux only)
Unit: MiB.
"""
import ctypes
ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(self._oOptions.sScratchRoot), None, None,
else:
# Convert to MB
# Round free space size
cMbFreeSpace = long(math.floor(cMbFreeSpace / self.kcMbScratchSpaceRounding)) * self.kcMbScratchSpaceRounding;
return cMbFreeSpace;
def _getScriptRev(self):
"""
The script (subversion) revision number.
"""
def _getPythonHexVersion(self):
"""
The python hex version number.
"""
if uHexVersion is None:
uHexVersion = (sys.version_info[0] << 24) | (sys.version_info[1] << 16) | (sys.version_info[2] << 8);
uHexVersion |= 0xf0;
return uHexVersion;
# @}
"""
Opens up a connection to the test manager.
Raises exception on failure.
"""
"""
Returns a sign-on parameter value as string.
Raises exception if the name is incorrect.
"""
def getPathState(self):
"""
Get the path to the state dir in the scratch area.
"""
return self._sScratchState;
def getPathScripts(self):
"""
Get the path to the scripts dir (TESTBOX_PATH_SCRIPTS) in the scratch area.
"""
return self._sScratchScripts;
def getPathSpill(self):
"""
Get the path to the spill dir (TESTBOX_PATH_SCRATCH) in the scratch area.
"""
return self._sScratchSpill;
def getPathBuilds(self):
"""
Get the path to the builds.
"""
def getTestBoxId(self):
"""
Get the TestBox ID for state saving purposes.
"""
return self._idTestBox;
def getTestBoxName(self):
"""
Get the TestBox name for state saving purposes.
"""
return self._sTestBoxName;
"""
Wipes the scratch directories and re-initializes them.
No exceptions raise, returns success indicator instead.
"""
if fUseTheForce is None:
"""
Callbacks + state for the cleanup.
"""
""" Logs error during shutil.rmtree operation. """
oRc = ErrorCallback();
#
# Cleanup.
#
try:
else:
raise Exception('Still exists after deletion, weird.');
if fUseTheForce is True \
try:
else:
if iRc != 0:
raise Exception('Still exists after forced deletion, weird^2.');
except:
else:
#
# Re-create the directories.
#
for sDir in [self._oOptions.sScratchRoot, self._sScratchSpill, self._sScratchScripts, self._sScratchState]:
try:
else:
"""
Worker for _maybeSignOn that does the actual signing on.
"""
# Reset the siged-on state.
self._idTestBox = None
# Assemble SIGN-ON request parameters and send the request.
dParams = {};
# Check response.
try:
except TestBoxException, err:
return False;
# Successfully signed on, update the state.
# Update the environment.
os.environ['TESTBOX_SCRATCH_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE);
# Set up the scratch area.
return True;
def _maybeSignOn(self):
"""
Check if Test Box parameters were changed
and do sign-in in case of positive result
"""
# Skip sign-on check if background command is currently in
# running state (avoid infinite signing on).
return None;
# Refresh sign-on parameters, changes triggers sign-on.
continue
if fNeedSignOn:
return None;
"""
Receive orders from Test Manager and execute them
"""
if self._fSignedOn:
while True:
# Make sure we're signed on before trying to do anything.
self._maybeSignOn();
while not self._fSignedOn:
self._maybeSignOn();
# Retrieve and handle command from the TM.
(oResponse, oConnection) = TestBoxConnection.requestCommandWithConnection(self._oOptions.sTestManagerUrl,
if oResponse is not None:
if oConnection is not None:
if oConnection.isConnected():
oConnection.close();
# Automatically reboot if scratch init fails.
% ( self._cReinitScratchErrors, ));
# delay a wee bit before looping.
## @todo We shouldn't bother the server too frequently. We should try combine the test reporting done elsewhere
# with the command retrieval done here. I believe tinderclient.pl is capable of doing that.
iFactor = 1;
iFactor = 4;
# Not reached.
def main():
"""
Main function a la C/C++. Returns exit code.
"""
#
# Parse arguments.
#
sDefTestRsrc = 'T:';
sDefBuilds = 'U:';
sDefTestRsrc = '/Volumes/testrsrc';
sDefBuilds = '/Volumes/builds';
else:
sDefTestRsrc = '/mnt/testrsrc';
sDefBuilds = '/mnt/builds';
class MyOptionParser(OptionParser):
""" We need to override the exit code on --help, error and so on. """
for sMixed, sDefault, sDesc in [('Builds', sDefBuilds, 'builds'), ('TestRsrc', sDefTestRsrc, 'test resources') ]:
help='The type of server, cifs or nfs. If empty (default), we won\'t try mount anything.');
help='The name of the server with the builds.');
help='The name of the builds share.');
dest="sTestManagerUrl",
help="Test Manager URL",
dest="sScratchRoot",
help="Path to the scratch directory",
default=None)
dest="sSystemUuid",
help="The system UUID of the testbox, used for uniquely identifiying the machine",
default=None)
help="Hardware virtualization available in the CPU");
help="Hardware virtualization not available in the CPU");
help="Nested paging is available");
help="Nested paging is not available");
help="Host can execute 64-bit guests");
help="Host cannot execute 64-bit guests");
help="I/O MMU available");
help="No I/O MMU available");
help="For the parent script, ignored.");
help = "Sets an environment variable. Can be repeated.");
# Check command line
if args != []:
parser.print_help();
return TBS_EXITCODE_SYNTAX;
if oOptions.sSystemUuid is not None:
return TBS_EXITCODE_SYNTAX;
print('Syntax error: Invalid server type "%s"' % (sType,));
return TBS_EXITCODE_SYNTAX;
#
# Instantiate the testbox script and start dispatching work.
#
try:
except TestBoxScriptException, oXcpt:
print('Error: %s' % (oXcpt,));
return TBS_EXITCODE_SYNTAX;
# Not supposed to get here...
return TBS_EXITCODE_FAILURE;
if __name__ == '__main__':