__copyright__ = \

__version__ = "$Revision$"

import sys

import os

from optparse import OptionParser

import time;

import zipfile;

g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

sys.path.append(g_ksTestManagerDir)

from common import utils;

from testmanager import config;

from testmanager.core.db import TMDatabaseConnection;

from testmanager.core.testset import TestSetData, TestSetLogic;

class FileArchiverBatchJob(object): # pylint: disable=R0903

"""

def __init__(self, oOptions):

"""

self.fVerbose = oOptions.fVerbose;

self.sSrcDir = config.g_ksFileAreaRootDir;

self.sDstDir = config.g_ksZipFileAreaRootDir;

#self.oTestSetLogic = TestSetLogic(TMDatabaseConnection(self.dprint if self.fVerbose else None));

self.oTestSetLogic = TestSetLogic(TMDatabaseConnection(None));

self.fDryRun = oOptions.fDryRun;

def dprint(self, sText):

""" Verbose output. """

if self.fVerbose:

print sText;

return True;

def warning(self, sText):

"""Prints a warning."""

print sText;

return True;

def _processTestSet(self, idTestSet, asFiles, sCurDir):

"""

sBaseFilename = os.path.join(sCurDir, 'TestSet-%d' % (idTestSet,));

if sBaseFilename[0:2] == ('.' + os.path.sep):

sBaseFilename = sBaseFilename[2:];

sSrcFileBase = os.path.join(self.sSrcDir, sBaseFilename + '-');

#

# Skip the file if the test set is still running.

# But delete them if the testset is not found.

#

oTestSet = self.oTestSetLogic.tryFetch(idTestSet);

if oTestSet is not None and sBaseFilename != oTestSet.sBaseFilename:

self.warning('TestSet %d: Deleting because sBaseFilename differs: "%s" (disk) vs "%s" (db)' \

% (idTestSet, sBaseFilename, oTestSet.sBaseFilename,));

oTestSet = None;

if oTestSet is not None:

if oTestSet.enmStatus == TestSetData.ksTestStatus_Running:

self.dprint('Skipping test set #%d, still running' % (idTestSet,));

return True;

#

# If we have a zip file already, don't try recreate it as we might

# have had trouble removing the source files.

#

sDstDirPath = os.path.join(self.sDstDir, sCurDir);

sZipFileNm = os.path.join(sDstDirPath, 'TestSet-%d.zip' % (idTestSet,));

if not os.path.exists(sZipFileNm):

#

# Create zip file with all testset files as members.

#

self.dprint('TestSet %d: Creating %s...' % (idTestSet, sZipFileNm,));

if not self.fDryRun:

if not os.path.exists(sDstDirPath):

os.makedirs(sDstDirPath, 0o755);

utils.noxcptDeleteFile(sZipFileNm + '.tmp');

oZipFile = zipfile.ZipFile(sZipFileNm + '.tmp', 'w', zipfile.ZIP_DEFLATED, allowZip64 = True);

for sFile in asFiles:

sSuff = os.path.splitext(sFile)[1];

if sSuff in [ '.png', '.webm', '.gz', '.bz2', '.zip', '.mov', '.avi', '.mpg', '.gif', '.jpg' ]:

## @todo Consider storing these files outside the zip if they are a little largish.

self.dprint('TestSet %d: Storing %s...' % (idTestSet, sFile));

oZipFile.write(sSrcFileBase + sFile, sFile, zipfile.ZIP_STORED);

else:

self.dprint('TestSet %d: Deflating %s...' % (idTestSet, sFile));

oZipFile.write(sSrcFileBase + sFile, sFile, zipfile.ZIP_DEFLATED);

oZipFile.close();

#

# .zip.tmp -> .zip.

#

utils.noxcptDeleteFile(sZipFileNm);

os.rename(sZipFileNm + '.tmp', sZipFileNm);

#else: Dry run.

else:

self.dprint('TestSet %d: zip file exists already (%s)' % (idTestSet, sZipFileNm,));

#

# Delete the files.

#

fRc = True;

if self.fVerbose:

self.dprint('TestSet %d: deleting file: %s' % (idTestSet, asFiles));

if not self.fDryRun:

for sFile in asFiles:

if utils.noxcptDeleteFile(sSrcFileBase + sFile) is False:

self.warning('TestSet %d: Failed to delete "%s" (%s)' % (idTestSet, sFile, sSrcFileBase + sFile,));

fRc = False;

return fRc;

def processDir(self, sCurDir):

"""

if self.fVerbose:

self.dprint('processDir: %s' % (sCurDir,));

#

# Sift thought the directory content, collecting subdirectories and

# sort relevant files by test set.

# Generally there will either be subdirs or there will be files.

#

asSubDirs = [];

dTestSets = {};

sCurPath = os.path.abspath(os.path.join(self.sSrcDir, sCurDir));

for sFile in os.listdir(sCurPath):

if os.path.isdir(os.path.join(sCurPath, sFile)):

if sFile not in [ '.', '..' ]:

asSubDirs.append(sFile);

elif sFile.startswith('TestSet-'):

# Parse the file name. ASSUMES 'TestSet-%d-filename' format.

iSlash1 = sFile.find('-');

iSlash2 = sFile.find('-', iSlash1 + 1);

if iSlash2 <= iSlash1:

self.warning('Bad filename (1): "%s"' % (sFile,));

continue;

try: idTestSet = int(sFile[(iSlash1 + 1):iSlash2]);

except:

self.warning('Bad filename (2): "%s"' % (sFile,));

if self.fVerbose:

self.dprint('\n'.join(utils.getXcptInfo(4)));

continue;

if idTestSet <= 0:

self.warning('Bad filename (3): "%s"' % (sFile,));

continue;

if iSlash2 + 2 >= len(sFile):

self.warning('Bad filename (4): "%s"' % (sFile,));

continue;

sName = sFile[(iSlash2 + 1):];

# Add it.

if idTestSet not in dTestSets:

dTestSets[idTestSet] = [];

asTestSet = dTestSets[idTestSet];

asTestSet.append(sName);

#

# Test sets.

#

fRc = True;

for idTestSet in dTestSets:

try:

if self._processTestSet(idTestSet, dTestSets[idTestSet], sCurDir) is not True:

fRc = False;

except:

self.warning('TestSet %d: Exception in _processTestSet:\n%s' % (idTestSet, '\n'.join(utils.getXcptInfo()),));

fRc = False;

#

# Sub dirs.

#

for sSubDir in asSubDirs:

if self.processDir(os.path.join(sCurDir, sSubDir)) is not True:

fRc = False;

#

# Try Remove the directory iff it's not '.' and it's been unmodified

# for the last 24h (race protection).

#

if sCurDir != '.':

try:

fpModTime = float(os.path.getmtime(sCurPath));

if fpModTime + (24*3600) <= time.time():

if utils.noxcptRmDir(sCurPath) is True:

self.dprint('Removed "%s".' % (sCurPath,));

except:

pass;

return fRc;

@staticmethod

def main():

""" C-style main(). """

#

# Parse options.

#

oParser = OptionParser();

oParser.add_option('-v', '--verbose', dest = 'fVerbose', action = 'store_true', default = False,

help = 'Verbose output.');

oParser.add_option('-q', '--quiet', dest = 'fVerbose', action = 'store_false', default = False,

help = 'Quiet operation.');

oParser.add_option('-d', '--dry-run', dest = 'fDryRun', action = 'store_true', default = False,

help = 'Dry run, do not make any changes.');

(oOptions, asArgs) = oParser.parse_args()

if asArgs != []:

oParser.print_help();

return 1;

#

# Do the work.

#

oBatchJob = FileArchiverBatchJob(oOptions);

fRc = oBatchJob.processDir('.');

return 0 if fRc is True else 1;

if __name__ == '__main__':

sys.exit(FileArchiverBatchJob.main());