Mercurial > repos > yufei-luo > s_mart
changeset 22:1e3f2c2657a3
Deleted selected files
| author | m-zytnicki |
|---|---|
| date | Mon, 29 Apr 2013 03:26:07 -0400 |
| parents | 91f8c4ce8c27 |
| children | 4541611db314 |
| files | commons/core/checker/AbstractChecker.py commons/core/checker/CheckerException.py commons/core/checker/CheckerUtils.py commons/core/checker/ConfigChecker.py commons/core/checker/ConfigException.py commons/core/checker/ConfigValue.py commons/core/checker/IChecker.py commons/core/checker/OldConfigChecker.py commons/core/checker/RepetException.py commons/core/checker/RepetException.pyc commons/core/checker/__init__.py commons/core/checker/__init__.pyc commons/core/checker/test/TestSuite_Checker.py commons/core/checker/test/Test_CheckerUtils.py commons/core/checker/test/Test_ConfigChecker.py commons/core/checker/test/Test_ConfigValue.py commons/core/checker/test/Test_F_ConfigChecker.py commons/core/checker/test/Test_OldConfigChecker.py commons/core/checker/test/__init__.py commons/core/launcher/JobScriptTemplate.py commons/core/launcher/JobScriptTemplateLight.py commons/core/launcher/JobScriptWithFilesCopyTemplate.py commons/core/launcher/Launcher.py commons/core/launcher/Launcher2.py commons/core/launcher/LauncherUtils.py commons/core/launcher/WriteScript.py commons/core/launcher/__init__.py commons/core/launcher/test/Test_Launcher.py commons/core/launcher/test/Test_Launcher2.py commons/core/launcher/test/Test_LauncherUtils.py commons/core/launcher/test/Test_WriteScript.py commons/core/launcher/test/__init__.py commons/core/launcher/test/expFiles/expJobScriptSQLiteWithFilesCopyTemplate.py commons/core/launcher/test/expFiles/expJobScriptTemplate.py commons/core/launcher/test/expFiles/expJobScriptTemplateLight.py commons/core/launcher/test/expFiles/expJobScriptTemplate_cmdWith2Lines.py commons/core/launcher/test/expFiles/expJobScriptWithFilesCopyTemplate.py commons/core/sql/DbFactory.py commons/core/sql/DbMySql.py commons/core/sql/DbSQLite.py commons/core/sql/ITableMapAdaptator.py commons/core/sql/ITableMatchAdaptator.py commons/core/sql/ITablePathAdaptator.py commons/core/sql/ITableSeqAdaptator.py commons/core/sql/ITableSetAdaptator.py commons/core/sql/Job.py commons/core/sql/JobAdaptator.py commons/core/sql/OldRepetDB.py commons/core/sql/RepetJob.py commons/core/sql/TableAdaptator.py commons/core/sql/TableBinPathAdaptator.py commons/core/sql/TableBinSetAdaptator.py commons/core/sql/TableJobAdaptator.py commons/core/sql/TableJobAdaptatorFactory.py commons/core/sql/TableMapAdaptator.py commons/core/sql/TableMatchAdaptator.py commons/core/sql/TablePathAdaptator.py commons/core/sql/TableSeqAdaptator.py commons/core/sql/TableSetAdaptator.py commons/core/sql/__init__.py commons/core/sql/test/TestSuite_sql.py commons/core/sql/test/Test_DbFactory.py commons/core/sql/test/Test_DbMySql.py commons/core/sql/test/Test_DbSQLite.py commons/core/sql/test/Test_F_JobAdaptator.py commons/core/sql/test/Test_F_TableJobAdaptator.py commons/core/sql/test/Test_Job.py commons/core/sql/test/Test_TableBinPathAdaptator.py commons/core/sql/test/Test_TableBinSetAdaptator.py commons/core/sql/test/Test_TableJobAdaptator.py commons/core/sql/test/Test_TableJobAdaptatorFactory.py commons/core/sql/test/Test_TableMapAdaptator.py commons/core/sql/test/Test_TableMatchAdaptator.py commons/core/sql/test/Test_TablePathAdaptator.py commons/core/sql/test/Test_TableSeqAdaptator.py commons/core/sql/test/Test_TableSetAdaptator.py commons/core/sql/test/Tst_F_RepetJob.py commons/core/sql/test/Tst_RepetJob.py commons/core/sql/test/__init__.py commons/core/stat/Stat.py commons/core/stat/__init__.py commons/core/stat/test/Test_F_Stat.py commons/core/stat/test/Test_Stat.py commons/core/stat/test/__init__.py commons/core/test/Test_LoggerFactory.py commons/core/test/__init__.py commons/core/tree/Tree.py commons/core/tree/__init__.py commons/core/tree/test/Test_Tree.py commons/core/tree/test/__init__.py commons/core/tree/test/treeTestSuite.py |
| diffstat | 80 files changed, 0 insertions(+), 18583 deletions(-) [+] |
line wrap: on
line diff
--- a/commons/core/checker/AbstractChecker.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,61 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - - -from commons.core.checker.IChecker import IChecker -from commons.core.LoggerFactory import LoggerFactory - - -## Enable a Logger in your Checker. -# -# Subclasses of AbstractChecker have a already a logger enabled (referenced by self._log attribute). Subclasses also already implements IChecker. -# All you have to do is to call __init__() method in your own constructor. -class AbstractChecker( IChecker ): - - ## Constructor - # - # @param logFileName name of log file where logger outputs - # - def __init__(self, logFileName): - self._log = LoggerFactory.createLogger(logFileName) - - - ## Set (change) default logger - # - # @param logger a new logger - # - def setLogger(self, logger): - self._log = logger - - - ## Return the logger instance - # - def getLogger(self): - return self._log
--- a/commons/core/checker/CheckerException.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,52 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - - -## Exception raised during check -# -# This class wraps Exception class -# -class CheckerException( Exception ): - - ## Constructor - # - # @param msg message embedded in Exception class - def __init__(self,msg=""): - self.messages = [] - self.msg = msg - Exception.__init__(self, msg) - - - def setMessages(self,lMessages): - self.messages = lMessages - - - def getMessages(self): - return self.messages
--- a/commons/core/checker/CheckerUtils.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,316 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - - -import os -import sys -import re -import glob -import ConfigParser -from ConfigParser import NoOptionError -from ConfigParser import NoSectionError -from commons.core.checker.CheckerException import CheckerException - - -## A set of static methods used to perform checks. -# -# -class CheckerUtils( object ): - - ## Check if blastName param is in ["blastn", "blastp", "blastx", "tblastn", "tblastx"] - # - # @param blastName name to check - # @return True if name is in list False otherwise - # - def isBlastNameNotInBlastValues( blastName ): - blastValuesSet = set( ["blastn", "blastp", "blastx", "tblastn", "tblastx"] ) - blastNameSet = set( [ blastName ] ) - return not blastNameSet.issubset( blastValuesSet ) - - isBlastNameNotInBlastValues = staticmethod( isBlastNameNotInBlastValues ) - - - ## Check if param is NOT "TRUE" and NOT false "FALSE" - # - # @param param str to check - # @return True if param is not eq to "TRUE" AND not eq to "FALSE", false otherwise - # - def isNotTRUEisNotFALSE( param ): - return param != "TRUE" and param != "FALSE" - - isNotTRUEisNotFALSE = staticmethod( isNotTRUEisNotFALSE ) - - - ## Check if resource (file or dir) do NOT exists - # - # @param resource file or dir to check - # @return True if resource exists False otherwise - # - def isRessourceNotExits( resource ): - return not os.path.exists( resource ) - - isRessourceNotExits = staticmethod( isRessourceNotExits ) - - - ## Check a specific E-value format: de-dd - # - # @param param E-value to check - # @return True if format is de-dd False otherwise - # - def isNotAeValueWithOneDigit2DecimalsAtLeast( param ): - # \d\d stands for 2 digits and more ??? - return not re.match( "\de\-\d\d", param ) - - isNotAeValueWithOneDigit2DecimalsAtLeast = staticmethod( isNotAeValueWithOneDigit2DecimalsAtLeast ) - - - ## Check a number format - # - # @param param value to check - # @return True if param is a number (d+) False otherwise - # - def isNotANumber( param ): - return not re.match( "\d+", param ) - - isNotANumber = staticmethod( isNotANumber ) - - - ## Check if an executable is in the user's PATH - # - # @param exeName name of the executable - # @return True if executable in user's PATH, False otherwise - # - def isExecutableInUserPath( exeName ): - dirPathList = os.environ["PATH"].split(":") - for dirPath in dirPathList: - if os.path.isdir( dirPath ): - try: - binPathList = glob.glob( dirPath + "/*" ) - except OSError, e: - continue - for binPath in binPathList: - bin = os.path.basename( binPath ) - if bin == exeName: - return True - return False - - isExecutableInUserPath = staticmethod( isExecutableInUserPath ) - - - ## Return the full path of a given executable - # - def getFullPathFromExecutable( exeName ): - lDirFromUserPath = os.environ["PATH"].split(":") - for dir in lDirFromUserPath: - if os.path.isdir( dir ): - try: - lExecutables = glob.glob( "%s/*" % ( dir ) ) - except OSError, e: - continue - for exe in lExecutables: - path, exe = os.path.split( exe ) - if exe == exeName: - return path - return "" - - getFullPathFromExecutable = staticmethod( getFullPathFromExecutable ) - - - #TODO: to remove ? - ## Check if a queue Name is valid. Warning: Only with the queue manager SGE - # - # @param fullQueueName name of the queue to test (with or without parameters) - # @return True if queue name is valid, False otherwise - # - def isQueueNameValid( fullQueueName ): - queueName = fullQueueName.split()[0] - if queueName == "none": - return True - queueFile = "queueName.txt" - if not CheckerUtils.isExecutableInUserPath( "qconf" ): - msg = "executable 'qconf' can't be found" - sys.stderr.write( "%s\n" % ( msg ) ) - return False - cmd = "qconf -sql > " + queueFile - os.system( cmd ) - queueFileHandler = open( queueFile, "r" ) - lQueueNames = queueFileHandler.readlines() - queueFileHandler.close() - os.remove( queueFile ) - queueNameValid = False - for qName in lQueueNames: - qName = qName.strip() - if qName == queueName: - queueNameValid = True - break - return queueNameValid - - isQueueNameValid = staticmethod( isQueueNameValid ) - - - ## Check if a string length is lower or equal than 15 - # - # @param strName any string - # @return True if string length is <= 15, False otherwise - # - def isMax15Char( strName ): - return (len(strName) <= 15 ) - - isMax15Char = staticmethod( isMax15Char ) - - - ## Check if a string is made with only alphanumeric or underscore character - # - # @param strName any string - # @return True if string is with alphanumeric or underscore, False otherwise - # - def isCharAlphanumOrUnderscore( strName ): - # authorized ALPHABET [a-z,A-Z,0-9,_] - p = re.compile('\W') - errList=p.findall(strName) - if len( errList ) > 0 : - return False - else: - return True - - isCharAlphanumOrUnderscore = staticmethod( isCharAlphanumOrUnderscore ) - - - ## Check if sectionName is in the configuration file - # - # @param config filehandle of configuration file - # @param sectionName string of section name to check - # @exception NoSectionError: if section not found raise a NoSectionError - # - def checkSectionInConfigFile( config, sectionName ): - if not (config.has_section(sectionName)): - raise NoSectionError(sectionName) - - checkSectionInConfigFile = staticmethod( checkSectionInConfigFile ) - - - ## Check if an option is in a specified section in the configuration file - # - # @param config filehandle of configuration file - # @param sectionName string of section name - # @param optionName string of option name to check - # @exception NoOptionError: if option not found raise a NoOptionError - # - def checkOptionInSectionInConfigFile( config, sectionName, optionName ): - config.get( sectionName, optionName ) - - checkOptionInSectionInConfigFile = staticmethod( checkOptionInSectionInConfigFile ) - - - ## Check version number coherency between configFile and CHANGELOG - # - # @param config ConfigParser Instance of configuration file - # @param changeLogFileHandle CHANGELOG file handle - # @exception NoOptionError: if option not found raise a NoOptionError - # - def checkConfigVersion( changeLogFileHandle, config ): - line = changeLogFileHandle.readline() - while not line.startswith("REPET release "): - line = changeLogFileHandle.readline() - numVersionChangeLog = line.split()[2] - - numVersionConfig = config.get("repet_env", "repet_version") - - if not numVersionChangeLog == numVersionConfig: - message = "*** Error: wrong config file version. Expected version num is " + numVersionChangeLog + " but actual in config file is " + numVersionConfig - raise CheckerException(message) - - checkConfigVersion = staticmethod( checkConfigVersion ) - - - ## Get version number from CHANGELOG - # - # @param changeLogFile CHANGELOG file name - # - def getVersionFromChangelogFile(changeLogFileName): - with open(changeLogFileName) as changeLogFileHandle: - line = changeLogFileHandle.readline() - while not line.startswith("REPET release "): - line = changeLogFileHandle.readline() - numVersionChangeLog = line.split()[2] - return numVersionChangeLog - - - getVersionFromChangelogFile = staticmethod( getVersionFromChangelogFile ) - - - ## Check if headers of an input file contain only alpha numeric characters and "_ : . -" - # - # @param fileHandler file handle - # @exception CheckerException if bad header raise a CheckerException - # - def checkHeaders( fileHandler ): - lHeaders = CheckerUtils._getHeaderFromFastaFile(fileHandler) - p = re.compile('[^a-zA-Z0-9_:\.\-]', re.IGNORECASE) - lWrongHeaders = [] - for header in lHeaders: - errList=p.findall(header) - if len( errList ) > 0 : - lWrongHeaders.append(header) - if lWrongHeaders != []: - exception = CheckerException() - exception.setMessages(lWrongHeaders) - raise exception - - checkHeaders = staticmethod( checkHeaders ) - - - def _getHeaderFromFastaFile( inFile ): - lHeaders = [] - while True: - line = inFile.readline() - if line == "": - break - if line[0] == ">": - lHeaders.append( line[1:-1] ) - return lHeaders - - _getHeaderFromFastaFile = staticmethod( _getHeaderFromFastaFile ) - - - ## Return True if an option is in a specified section in the configuration file, False otherwise - # - # @param config handler of configuration file - # @param sectionName string of section name - # @param optionName string of option name to check - # - def isOptionInSectionInConfig( configHandler, section, option ): - try: - CheckerUtils.checkOptionInSectionInConfigFile( configHandler, section, option ) - except NoOptionError: - return False - return True - - isOptionInSectionInConfig = staticmethod( isOptionInSectionInConfig )
--- a/commons/core/checker/ConfigChecker.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,226 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - - -import re -import sys -from commons.core.utils.RepetConfigParser import RepetConfigParser -from commons.core.checker.ConfigValue import ConfigValue -from commons.core.checker.IChecker import IChecker -from commons.core.checker.RepetException import RepetException -from commons.core.utils.FileUtils import FileUtils - - -class Rule(object): - - def __init__(self, mandatory= False, isPattern=False, type="", set=(), help =""): - self.mandatory = mandatory - self.isPattern = isPattern - self.type = type - self.set = set - self.help = help - -class ConfigRules(object): - - def __init__(self, configName = "", configDescription = ""): - self.configName = configName - self.configDescription = configDescription - self.dRules4Sections={} - - def _addRule(self, section, option="DEFAULT", mandatory=False, isPattern=False, type="", set=(), help =""): - if not self.dRules4Sections.has_key(section): - self.dRules4Sections[section] = {} - self.dRules4Sections[section][option]=Rule(mandatory, isPattern, type.lower(), set) - - def addRuleSection(self, section, mandatory=False, isPattern=False, help = ""): - self._addRule(section = section, option = "DEFAULT", mandatory = mandatory, isPattern = isPattern, help = "") - - def addRuleOption(self, section, option, mandatory=False, isPattern=False, type="", set=(), help = ""): - self._addRule(section = section, option = option, mandatory = mandatory, isPattern = isPattern, type = type, set=set , help = "") - - def isSectionMandatory(self, section): - if self.dRules4Sections.has_key(section): - if self.dRules4Sections[section].has_key("DEFAULT"): - return self.dRules4Sections[section]["DEFAULT"].mandatory - return False - - def isOptionMandatory(self, section, option): - if self.dRules4Sections.has_key(section): - if self.dRules4Sections[section].has_key(option): - return self.dRules4Sections[section][option].mandatory - return False - - def getRule(self, section, option): - if self.dRules4Sections.has_key(section): - if self.dRules4Sections[section].has_key(option): - return self.dRules4Sections[section][option] - return None - -class ConfigChecker(IChecker): - - def __init__ (self, cfgFileName, iCfgRules): - self._configFileName = cfgFileName - self._iConfigRules = iCfgRules - self._iRawConfig = ConfigValue() - self._iExtendedConfigRules = ConfigRules() - - def readConfigFile(self): - iConfig = RepetConfigParser() - try: - iConfig.readfp(open(self._configFileName)) - return iConfig -# TODO USE OF CONFIG ERROR -# if DuplicateSectionError: -# raise Exception ("Duplicate section exist in config file %s" %(self._configFileName )) - except : - raise RepetException ("Unexpected error: %s" % sys.exc_info()[0]) - - def setRawConfig(self, iConfig ): - for sectionName in iConfig.sections(): - for optionName in iConfig.options(sectionName): - optionValue = iConfig.get(sectionName, optionName) - self._iRawConfig.set(sectionName, optionName, optionValue) - - def getOptionValueAccordingRule(self, iConfig, sectionName, optionName): - optionRule = self._iExtendedConfigRules.getRule(sectionName, optionName) - if optionRule == None : - return iConfig.get(sectionName, optionName) - - if optionRule.type == "int": - optionValue = iConfig.getint(sectionName, optionName) - elif optionRule.type == "float": - optionValue = iConfig.getfloat(sectionName, optionName) - elif optionRule.type == "bool" or optionRule.type == "boolean": - optionValue = iConfig.getboolean(sectionName, optionName) - else: - optionValue = iConfig.get(sectionName, optionName) - if optionRule.set!=() and not(optionValue in optionRule.set): - #TODO : test and fix - raise RepetException ("value must be in %s " % set.__repr__()) - - return optionValue - - def setConfig(self, iConfig ): - config = ConfigValue() - valueErr = "" - for sectionName in iConfig.sections(): - for optionName in iConfig.options(sectionName): - try: - optionValue = self.getOptionValueAccordingRule(iConfig, sectionName, optionName ) - config.set(sectionName, optionName, optionValue) - except RepetException, re : - #TODO : test and fix - valueErr += "\n - [%s]: %s %s" % re.getMessage() - if valueErr == "": - self._iRawConfig = config - else: - raise RepetException ("Following errors occurs:%s\n" %valueErr) - - def checkIfExistsConfigFile (self): - if not (FileUtils.isRessourceExists(self._configFileName)): - raise RepetException("CONFIG FILE not found - '%s'" % self._configFileName) - - def checkMandatorySections (self): - missingSection = "" - for sectionName in self._iExtendedConfigRules.dRules4Sections.keys(): - if self._iExtendedConfigRules.isSectionMandatory(sectionName) and not self._iRawConfig.has_section(sectionName): - missingSection += "\n - %s" %(sectionName) - if missingSection != "": - raise RepetException ("Error in configuration file %s, following sections are missing:%s\n" % (self._configFileName, missingSection)) - - def checkMandatoryOptions (self): - missingOption = "" - for sectionName in self._iExtendedConfigRules.dRules4Sections.keys(): - if self._iExtendedConfigRules.isSectionMandatory(sectionName) or self._iRawConfig.has_section(sectionName) : - dRules4OptionsOfThisSection = self._iExtendedConfigRules.dRules4Sections[sectionName] - for optionName in dRules4OptionsOfThisSection.keys(): - if optionName != "DEFAULT" and self._iExtendedConfigRules.isOptionMandatory(sectionName, optionName) and not self._iRawConfig.has_option(sectionName, optionName): - missingOption += "\n - [%s]: %s" % (sectionName, optionName) - if missingOption != "": - raise RepetException ("Error in configuration file %s, following options are missing: %s\n" % (self._configFileName, missingOption)) - - def getSectionNamesAccordingPatternRules (self, sectionWordOrPattern, isPattern): - lSectionsFoundAccordingPatternRules=[] - if isPattern == False: - if self._iRawConfig.has_section(sectionWordOrPattern): - lSectionsFoundAccordingPatternRules.append(sectionWordOrPattern) - else: - for sectionName in self._iRawConfig.sections(): - if re.search(sectionWordOrPattern, sectionName, re.IGNORECASE): - lSectionsFoundAccordingPatternRules.append(sectionName) - return lSectionsFoundAccordingPatternRules - - def getOptionsNamesAccordingPatternRules(self, sectionName, optionWordOrPattern, isPattern): - lOptionsFoundAccordingPatternRules=[] - if isPattern == False: - if self._iRawConfig.has_option(sectionName, optionWordOrPattern): - lOptionsFoundAccordingPatternRules.append(optionWordOrPattern) - else : - for optionName in self._iRawConfig.options(sectionName): - if re.search(optionWordOrPattern, optionName, re.IGNORECASE)!= None: - lOptionsFoundAccordingPatternRules.append(optionName) - return lOptionsFoundAccordingPatternRules - - def extendConfigRulesWithPatternRules(self): - for sectionName in self._iConfigRules.dRules4Sections.keys(): - dRules4OptionsOfThisSection = self._iConfigRules.dRules4Sections[sectionName] - lRawSections=[] - if dRules4OptionsOfThisSection.has_key("DEFAULT"): - mandatorySection = dRules4OptionsOfThisSection["DEFAULT"].mandatory - isPatternSection = dRules4OptionsOfThisSection["DEFAULT"].isPattern - lRawSections=self.getSectionNamesAccordingPatternRules(sectionName, isPatternSection) - for rawSectionName in lRawSections: - self._iExtendedConfigRules.addRuleSection(rawSectionName, "DEFAULT", mandatorySection ) - if mandatorySection and (len(lRawSections)==0): - self._iExtendedConfigRules.addRuleSection(sectionName, "DEFAULT", mandatorySection ) - else: - lRawSections.append(sectionName) - for optionName in dRules4OptionsOfThisSection.keys(): - setOption = dRules4OptionsOfThisSection[optionName].set - isPatternOption = dRules4OptionsOfThisSection[optionName].isPattern - mandatoryOption = dRules4OptionsOfThisSection[optionName].mandatory - typeOption = dRules4OptionsOfThisSection[optionName].type - if optionName != "DEFAULT": - for rawSectionName in lRawSections: - lRawOptions=self.getOptionsNamesAccordingPatternRules(rawSectionName, optionName, isPatternOption) - for rawOptionName in lRawOptions: - self._iExtendedConfigRules.addRuleOption(rawSectionName, rawOptionName, mandatoryOption, False, typeOption, setOption) - if mandatoryOption and (len(lRawOptions)==0): - self._iExtendedConfigRules.addRuleOption(rawSectionName, optionName, mandatoryOption, False, typeOption, setOption) - - def getConfig(self): - self.checkIfExistsConfigFile() - iConfig = self.readConfigFile() - self.setRawConfig(iConfig) - self.extendConfigRulesWithPatternRules() - self.checkMandatorySections() - self.checkMandatoryOptions() - self.setConfig(iConfig) - return self._iRawConfig \ No newline at end of file
--- a/commons/core/checker/ConfigException.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,53 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - -from commons.core.checker.RepetException import RepetException - -## A exception raised by check() method of class ConfigChecker -# -# This class allow storage of multiple messages (see messages attribute). -# Example: use one instance of ConfigException class for one section in configuration file. -# All messages relatives to this section are stored in messages attribute. -class ConfigException( RepetException ): - - ## Constructor - # - # @param msg message embedded in Exception class - # - def __init__(self, msg, messages = []): - RepetException.__init__(self, msg) - self.messages = messages - - def getMessages(self): - return self.messages - - def setMessages(self, messages): - self.messages = messages -
--- a/commons/core/checker/ConfigValue.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,70 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - - -class ConfigValue(object): - - def __init__(self): - self.dOptionsValues4Sections={} - - def has_section(self,sectionName): - return self.dOptionsValues4Sections.has_key(sectionName) - - def has_option(self, sectionName, optionName): - isOptionExist = False - if self.has_section(sectionName): - isOptionExist = self.dOptionsValues4Sections[sectionName].has_key(optionName) - return isOptionExist - - def sections(self): - lSectionsKeys = self.dOptionsValues4Sections.keys() - return lSectionsKeys - - def options(self, sectionName): - lOptionsKeys = [] - if self.has_section(sectionName): - lOptionsKeys = self.dOptionsValues4Sections[sectionName].keys() - return lOptionsKeys - - def get(self, sectionName, optionName): - if self.has_option(sectionName, optionName): - return self.dOptionsValues4Sections[sectionName][optionName] - return None - - def set(self, sectionName, optionName, optionValue): - if not (self.has_section(sectionName)): - self.dOptionsValues4Sections[sectionName] = {} - self.dOptionsValues4Sections[sectionName][optionName] = optionValue - - def setdOptionsValues4Sections(self, dOptionsValues4Sections): - self.dOptionsValues4Sections = dOptionsValues4Sections - - def __eq__(self, o): - return self.dOptionsValues4Sections == o.dOptionsValues4Sections
--- a/commons/core/checker/IChecker.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,45 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - - -## Interface for a checker -# -# This class emulates an interface for a checker. -# -# All checkers are subclasses of IChecker. -# -class IChecker( object ): - - ## perform check, raise a CheckerException if error occurred - # - # @param arg a collecting parameter: put here all you need to perform check - # - def check(self, arg=""): - pass
--- a/commons/core/checker/OldConfigChecker.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,101 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - - -import ConfigParser -from ConfigParser import NoOptionError -from commons.core.checker.IChecker import IChecker -from commons.core.checker.ConfigException import ConfigException - - -## A checker for a configuration file -# -# -# A configuration file is formatted as follow: -# -# [section1] -# -# option_name1: option_value1 -# -# option_name2: option_value2 -# -# option_name3: option_value3 -# -# [section2] -# -# ... -# -# -# This class performs 3 checkes on a configuration file: -# -# (i) check if file exists -# -# (ii) check if section exists -# -# (iii) check if option exists -# -class ConfigChecker( IChecker ): - - ## Constructor A checker for configuration file. - # - # @param sectionName name of section to check in configuration file - # @param optionsDict dictionary with option(s) to check as keys and empty strings ("") as values - def __init__ (self, sectionName, optionsDict): - self._sectionName = sectionName - self._optionsDict = optionsDict - - - ## Perform 3 checks : file exists, sections exists, option exists - # - # @param configFile configuration file to check - # @exception ConfigException with a list of messages - def check (self, configFile): - config = ConfigParser.ConfigParser() - msg = [] - try: - config.readfp( open(configFile) ) - except IOError, e: - msg.append("CONFIG FILE not found - " + e.message) - raise ConfigException("", msg) - - if not (config.has_section(self._sectionName)): - msg.append("[" + self._sectionName + "]" + " section not found - ") - raise ConfigException("", msg) - - isExceptionOccured = False - for key in self._optionsDict.keys(): - try: - self._optionsDict[key] = config.get(self._sectionName, key) - except NoOptionError, e: - msg.append("[" + self._sectionName + "]" + " - " + e.message) - isExceptionOccured = True - - if (isExceptionOccured): - raise ConfigException("", msg)
--- a/commons/core/checker/RepetException.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,51 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - - -class RepetException(Exception): - - def __init__(self, msg): - Exception.__init__(self) - self._message = msg - - def __str__(self): - return self._message - - def getMessage(self): - return self._message - - def setMessage(self, msg): - self._message = msg - - -class RepetDataException(RepetException): - - def __init__(self, msg): - RepetException.__init__(self, msg) \ No newline at end of file
--- a/commons/core/checker/test/TestSuite_Checker.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,51 +0,0 @@ -#!/usr/bin/env python - -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - - -import sys -import unittest -import Test_CheckerUtils -import Test_ConfigChecker - - -def main(): - - TestSuite_Checker = unittest.TestSuite() - - TestSuite_Checker.addTest( unittest.makeSuite( Test_CheckerUtils.Test_CheckerUtils, "test" ) ) - TestSuite_Checker.addTest( unittest.makeSuite( Test_ConfigChecker.Test_ConfigChecker, "test" ) ) - - runner = unittest.TextTestRunner( sys.stderr, 2, 2 ) - runner.run( TestSuite_Checker ) - -if __name__ == "__main__": - main()
--- a/commons/core/checker/test/Test_CheckerUtils.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,535 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - - -import unittest -import os -import ConfigParser -from commons.core.checker.CheckerUtils import CheckerUtils -from commons.core.checker.CheckerException import CheckerException -from ConfigParser import NoOptionError -from ConfigParser import NoSectionError - -class Test_CheckerUtils( unittest.TestCase ): - - def setUp(self): - self.queueFileName = "queueName.txt" - self.configFileName = "dummyConfig.cfg" - - def tearDown(self): - if os.path.exists(self.queueFileName): - os.remove(self.queueFileName) - if os.path.exists(self.configFileName): - os.remove(self.configFileName) - - def test_isBlastNameInBlastValues( self ): - correctValueList = [ "blastn", "blastp", "blastx", "tblastn", "tblastx" ] - for value in correctValueList: - self.assertFalse( CheckerUtils.isBlastNameNotInBlastValues( value ) ) - - incorrectValueList = [ "badbalst", "wublast" ] - for value in incorrectValueList: - self.assertTrue( CheckerUtils.isBlastNameNotInBlastValues( value ) ) - - def test_isNotTRUEisNotFALSE( self ): - correctValueList = [ "TRUE", "FALSE" ] - for value in correctValueList: - self.assertFalse( CheckerUtils.isNotTRUEisNotFALSE( value ) ) - - incorrectValueList = [ "True", "False" ] - for value in incorrectValueList: - self.assertTrue( CheckerUtils.isNotTRUEisNotFALSE( value ) ) - - def test_isRessourceNotExists( self ): - fileName = "dummyFile.txt" - self.assertTrue( CheckerUtils.isRessourceNotExits( fileName ) ) - os.system( "touch %s" % ( fileName ) ) - self.assertFalse( CheckerUtils.isRessourceNotExits( fileName ) ) - os.remove( fileName ) - - def test_isNotAeValueWithOneDigit2DecimalsAtLeast( self ): - correctEValueList = [ "5e-32", "7e-45", "1e-2122", "9e-32" ] - for value in correctEValueList: - self.assertFalse( CheckerUtils.isNotAeValueWithOneDigit2DecimalsAtLeast( value ) ) - - incorrecEValueStr = [ "10e-32", "2e-3", "2e-2", "1", "cxhhe" ] - for value in incorrecEValueStr: - self.assertTrue( CheckerUtils.isNotAeValueWithOneDigit2DecimalsAtLeast( value ) ) - - def test_isNotADigit( self ): - correctDigitList = [ "1", "21", "345" ] - for value in correctDigitList: - self.assertFalse( CheckerUtils.isNotANumber( value ) ) - - incorrectDigitList = [ "A", "dfdf", "((((" ] - for value in incorrectDigitList: - self.assertTrue( CheckerUtils.isNotANumber( value ) ) - - def test_isExecutableInUserPath( self ): - exeName = "ls" - self.assertTrue( CheckerUtils.isExecutableInUserPath( exeName ) ) - - exeName = "hsvegdtefaobfbcheta" - self.assertFalse( CheckerUtils.isExecutableInUserPath( exeName ) ) - - def test_isExecutableInUserPath_case_endsWith( self ): - exeName = "s" - self.assertFalse( CheckerUtils.isExecutableInUserPath( exeName ) ) - - def test_isQueueNameValid_valid(self): - cmd = "qconf -sql > " + self.queueFileName - os.system(cmd) - f = open(self.queueFileName, "r") - lQueueNames = f.readlines() - f.close() - if lQueueNames == []: - print "Warning you need SGE installed\n" - else: - self.assertTrue(CheckerUtils.isQueueNameValid(lQueueNames[0].strip())) - - def test_isQueueNameValid_not_valid(self): - cmd = "qconf -sql > " + self.queueFileName - os.system(cmd) - f = open(self.queueFileName, "r") - lQueueNames = f.readlines() - f.close() - if lQueueNames == []: - print "Warning you need SGE installed\n" - else: - self.assertFalse(CheckerUtils.isQueueNameValid("dummyQueueName")) - - def test_isQueueNameValid_valid_with_parameter(self): - cmd = "qconf -sql > " + self.queueFileName - os.system(cmd) - f = open(self.queueFileName, "r") - lQueueNames = f.readlines() - f.close() - if lQueueNames == []: - print "Warning you need SGE installed\n" - else: - queueNameWithParameter = lQueueNames[0].strip() + " test=TRUE" - self.assertTrue(CheckerUtils.isQueueNameValid(queueNameWithParameter)) - - def test_isCharAlphanumOrUnderscore_valid(self): - project_name = "Project_Name" - self.assertTrue(CheckerUtils.isCharAlphanumOrUnderscore(project_name)) - - def test_isCharAlphanumOrUnderscore_not_valid(self): - project_name = "Project_Name," - self.assertFalse(CheckerUtils.isCharAlphanumOrUnderscore(project_name)) - - def test_isMax15Char_valid(self): - project_name = "Project_Name" - self.assertTrue(CheckerUtils.isMax15Char(project_name)) - - def test_isMax15Char_not_valid(self): - project_name = "Project_Name_tooLong" - self.assertFalse(CheckerUtils.isMax15Char(project_name)) - - def test_checkSectionInConfigFile_empty_section(self): - sectionName = "sectionName" - line = "[" + sectionName + "]" - configFile = open(self.configFileName, "w") - configFile.write(line) - configFile.close() - configFile = open(self.configFileName, "r") - - config = ConfigParser.ConfigParser() - config.readfp(configFile) - - isSectionExceptionRaised = False - try: - CheckerUtils.checkSectionInConfigFile(config, sectionName) - except NoSectionError: - isSectionExceptionRaised = True - configFile.close() - self.assertFalse(isSectionExceptionRaised) - - def test_checkSectionInConfigFile(self): - sectionName = "sectionName" - lines = "[" + sectionName + "]\ndummyOption : dummyOption\n" - configFile = open(self.configFileName, "w") - configFile.write(lines) - configFile.close() - configFile = open(self.configFileName, "r") - - config = ConfigParser.ConfigParser() - config.readfp(configFile) - - isSectionExceptionRaised = False - try: - CheckerUtils.checkSectionInConfigFile(config, sectionName) - except NoSectionError: - isSectionExceptionRaised = True - configFile.close() - self.assertFalse(isSectionExceptionRaised) - - def test_checkSectionInConfigFile_not_valid(self): - sectionName = "sectionName" - line = " " - configFile = open(self.configFileName, "w") - configFile.write(line) - configFile.close() - configFile = open(self.configFileName, "r") - - config = ConfigParser.ConfigParser() - config.readfp(configFile) - - isSectionExceptionRaised = False - try: - CheckerUtils.checkSectionInConfigFile(config, sectionName) - except NoSectionError: - isSectionExceptionRaised = True - configFile.close() - self.assertTrue(isSectionExceptionRaised) - - def test_checkOptionInSectionInConfigFile(self): - optionName = "optionName" - sectionName = "sectionName" - configFile = open(self.configFileName, "w") - configFile.write("[" + sectionName + "]\n") - configFile.write(optionName + ": ") - configFile.close() - configFile = open(self.configFileName, "r") - - config = ConfigParser.ConfigParser() - config.readfp(configFile) - - isSectionExceptionRaised = False - try: - CheckerUtils.checkOptionInSectionInConfigFile(config, sectionName, optionName) - except NoOptionError: - isSectionExceptionRaised = True - configFile.close() - self.assertFalse(isSectionExceptionRaised) - - def test_checkOptionInSectionInConfigFile_not_present(self): - optionName = " " - sectionName = "sectionName" - configFile = open(self.configFileName, "w") - configFile.write("[" + sectionName + "]\n") - configFile.write(optionName) - configFile.close() - configFile = open(self.configFileName, "r") - - config = ConfigParser.ConfigParser() - config.readfp(configFile) - - isSectionExceptionRaised = False - try: - CheckerUtils.checkOptionInSectionInConfigFile(config, sectionName, optionName) - except NoOptionError: - isSectionExceptionRaised = True - configFile.close() - self.assertTrue(isSectionExceptionRaised) - - def test_checkOptionInSectionInConfigFile_not_in_the_right_section(self): - optionName = "optionName" - rightSection = "dummySection2" - configFile = open(self.configFileName, "w") - configFile.write("[dummySection1]\n") - configFile.write(optionName + ": \n") - configFile.write("[" + rightSection + "]") - configFile.close() - configFile = open(self.configFileName, "r") - - config = ConfigParser.ConfigParser() - config.readfp(configFile) - - isSectionExceptionRaised = False - try: - CheckerUtils.checkOptionInSectionInConfigFile(config, rightSection, optionName) - except NoOptionError: - isSectionExceptionRaised = True - configFile.close() - self.assertTrue(isSectionExceptionRaised) - - def test_checkConfigVersion_different_version_between_changeLog_and_configFile(self): - changeLogFileName = "dummyCHANGELOG" - - self._writeChangeLogFile(changeLogFileName) - self._writeConfigFile("repet_version: 1.3.5\n") - - changeLogFileHandler = open(changeLogFileName, "r") - configFileHandler = open(self.configFileName, "r") - config = ConfigParser.ConfigParser() - config.readfp(configFileHandler) - - self.assertRaises(CheckerException, CheckerUtils.checkConfigVersion, changeLogFileHandler, config) - - changeLogFileHandler.close() - configFileHandler.close() - os.remove(changeLogFileName) - - def test_checkConfigVersion_same_version_between_changeLog_and_configFile(self): - changeLogFileName = "dummyCHANGELOG" - self._writeChangeLogFile(changeLogFileName) - self._writeConfigFile("repet_version: 1.3.6\n") - - changeLogFileHandler = open(changeLogFileName, "r") - configFileHandler = open(self.configFileName, "r") - config = ConfigParser.ConfigParser() - config.readfp(configFileHandler) - - isCheckerExceptionRaised = False - try: - CheckerUtils.checkConfigVersion(changeLogFileHandler, config) - except CheckerException: - isCheckerExceptionRaised = True - self.assertFalse(isCheckerExceptionRaised) - - changeLogFileHandler.close() - configFileHandler.close() - os.remove(changeLogFileName) - - def test_getVersionFromChangelogFile(self): - changeLogFileName = "dummyCHANGELOG" - self._writeChangeLogFile(changeLogFileName) - obsVer = CheckerUtils.getVersionFromChangelogFile(changeLogFileName) - expVer = "1.3.6" - - self.assertEquals(obsVer, expVer) - - os.remove(changeLogFileName) - - def test_checkConfigVersion_check_exception_message(self): - changeLogFileName = "dummyCHANGELOG" - self._writeChangeLogFile(changeLogFileName) - self._writeConfigFile("repet_version: 1.3.5\n") - - changeLogFileHandler = open(changeLogFileName, "r") - configFileHandler = open(self.configFileName, "r") - config = ConfigParser.ConfigParser() - config.readfp(configFileHandler) - - checkerExceptionInstance = None - try: - CheckerUtils.checkConfigVersion(changeLogFileHandler, config) - except CheckerException, e: - checkerExceptionInstance = e - - expMessage = "*** Error: wrong config file version. Expected version num is 1.3.6 but actual in config file is 1.3.5" - obsMessage = checkerExceptionInstance.msg - - self.assertEquals(expMessage, obsMessage) - - def test_checkHeaders_first_header_with_question_mark(self): - fastaFileName = "dummyFasta.fa" - self._writeFastaFile_first_header_with_question_mark(fastaFileName) - fastaFileHandler = open(fastaFileName,"r") - isExceptionRaised = False - try: - CheckerUtils.checkHeaders(fastaFileHandler) - except CheckerException, e: - isExceptionRaised = True - - fastaFileHandler.close() - os.remove(fastaFileName) - - self.assertTrue(isExceptionRaised) - - expMessages = ["DmelChr4_Bla?ster_Piler_0.0_Map_3"] - obsMessages = e.getMessages() - - self.assertEquals(expMessages, obsMessages) - - def test_checkHeaders_with_2_wrong_headers(self): - fastaFileName = "dummyFasta.fa" - self._writeFastaFile_with_2_wrong_headers(fastaFileName) - fastaFileHandler = open(fastaFileName,"r") - isExceptionRaised = False - try: - CheckerUtils.checkHeaders(fastaFileHandler) - except CheckerException, e: - isExceptionRaised = True - - fastaFileHandler.close() - os.remove(fastaFileName) - - self.assertTrue(isExceptionRaised) - - expMessages = ["DmelChr4_Bla?ster_Piler_0.0_Map_3","DmelChr4_Bla!ster_Piler_1.0_Map_9"] - obsMessages = e.getMessages() - - self.assertEquals(expMessages, obsMessages) - - def test_checkHeaders_with_pipe(self): - fastaFileName = "dummyFasta.fa" - self._writeFastaFile_with_pipe(fastaFileName) - fastaFileHandler = open(fastaFileName,"r") - isExceptionRaised = False - try: - CheckerUtils.checkHeaders(fastaFileHandler) - except CheckerException, e: - isExceptionRaised = True - - fastaFileHandler.close() - os.remove(fastaFileName) - - self.assertTrue(isExceptionRaised) - - expMessages = ["DmelC|hr4_Blas-ter_Piler_1.0_Map_9"] - obsMessages = e.getMessages() - - self.assertEquals(expMessages, obsMessages) - - def test_checkHeaders_with_equal(self): - fastaFileName = "dummyFasta.fa" - self._writeFastaFile_with_equal(fastaFileName) - fastaFileHandler = open(fastaFileName,"r") - isExceptionRaised = False - try: - CheckerUtils.checkHeaders(fastaFileHandler) - except CheckerException, e: - isExceptionRaised = True - - fastaFileHandler.close() - os.remove(fastaFileName) - - self.assertTrue(isExceptionRaised) - - expMessages = ["DmelC:hr4_Blas=ter_Piler_1.0_Map_9"] - obsMessages = e.getMessages() - - self.assertEquals(expMessages, obsMessages) - - def test_checkHeaders_all_headers_ok(self): - fastaFileName = "dummyFasta.fa" - self._writeFastaFile_all_headers_ok(fastaFileName) - fastaFileHandler = open(fastaFileName,"r") - isExceptionRaised = False - try: - CheckerUtils.checkHeaders(fastaFileHandler) - except CheckerException: - isExceptionRaised = True - - fastaFileHandler.close() - os.remove(fastaFileName) - - self.assertFalse(isExceptionRaised) - - def _writeFastaFile_first_header_with_question_mark(self, fastaFileName): - fastaFileHandler = open(fastaFileName, "w") - fastaFileHandler.write(">DmelChr4_Bla?ster_Piler_0.0_Map_3\n") - fastaFileHandler.write("ACCAAAGACACTAGAATAACAAGATGCGTAACGCCATACGATTTTTTGGCACACTATTTT\n") - fastaFileHandler.write("TTCGCCGTGGCTCTAGAGGTGGCTCCAGGCTCTCTCGAATTTTTGTTAGAGAGCGAGAGA\n") - fastaFileHandler.write("GCTGAGAGCGCTACAGCGAACAGCTCTTTTCTACACATAAAGTGATAGCAGACAACTGTA\n") - fastaFileHandler.write("TGTGTGCACACGTGTGCTCATGCATTGTAAATTTGACAAAATATGCCCTTCACCTTCAAA\n") - fastaFileHandler.write(">DmelChr4_Blaster_Piler_1.0_Map_9\n") - fastaFileHandler.write("AGTTTAAAAACCAAAGACACTAGAATAACAAGATGCGTAACGGCCATACATTGGTTTGGC\n") - fastaFileHandler.write("ACTATGCAGCCACTTTTTTGGTGACGGCCAAAATTACTCTCTTTCCGCTCACTCCCGCTG\n") - fastaFileHandler.write("AGAGCGTAAGAAATCTAAAAATATAATTTGCTTGCTTGTGTGAGTAAAAACAAGAGACGA\n") - fastaFileHandler.write("GAACGCGTATAAGTGTGCGTGTTGTGCTAGAAGACGATTTTCGGGACCGAAATCAATTCT\n") - fastaFileHandler.write("GATCGAAGAAACGAATTTACATGGTACATATTAGGGTAGTTTTTGCCAATTTCCTAGCAA\n") - fastaFileHandler.close() - - def _writeFastaFile_with_2_wrong_headers(self, fastaFileName): - fastaFileHandler = open(fastaFileName, "w") - fastaFileHandler.write(">DmelChr4_Bla?ster_Piler_0.0_Map_3\n") - fastaFileHandler.write("ACCAAAGACACTAGAATAACAAGATGCGTAACGCCATACGATTTTTTGGCACACTATTTT\n") - fastaFileHandler.write("TTCGCCGTGGCTCTAGAGGTGGCTCCAGGCTCTCTCGAATTTTTGTTAGAGAGCGAGAGA\n") - fastaFileHandler.write("GCTGAGAGCGCTACAGCGAACAGCTCTTTTCTACACATAAAGTGATAGCAGACAACTGTA\n") - fastaFileHandler.write("TGTGTGCACACGTGTGCTCATGCATTGTAAATTTGACAAAATATGCCCTTCACCTTCAAA\n") - fastaFileHandler.write(">DmelChr4_Bla!ster_Piler_1.0_Map_9\n") - fastaFileHandler.write("AGTTTAAAAACCAAAGACACTAGAATAACAAGATGCGTAACGGCCATACATTGGTTTGGC\n") - fastaFileHandler.write("ACTATGCAGCCACTTTTTTGGTGACGGCCAAAATTACTCTCTTTCCGCTCACTCCCGCTG\n") - fastaFileHandler.write("AGAGCGTAAGAAATCTAAAAATATAATTTGCTTGCTTGTGTGAGTAAAAACAAGAGACGA\n") - fastaFileHandler.write("GAACGCGTATAAGTGTGCGTGTTGTGCTAGAAGACGATTTTCGGGACCGAAATCAATTCT\n") - fastaFileHandler.write("GATCGAAGAAACGAATTTACATGGTACATATTAGGGTAGTTTTTGCCAATTTCCTAGCAA\n") - fastaFileHandler.close() - - def _writeFastaFile_all_headers_ok(self, fastaFileName): - fastaFileHandler = open(fastaFileName, "w") - fastaFileHandler.write(">DmelChr4_Blaster_Piler_0.0_Map_3\n") - fastaFileHandler.write("ACCAAAGACACTAGAATAACAAGATGCGTAACGCCATACGATTTTTTGGCACACTATTTT\n") - fastaFileHandler.write("TTCGCCGTGGCTCTAGAGGTGGCTCCAGGCTCTCTCGAATTTTTGTTAGAGAGCGAGAGA\n") - fastaFileHandler.write("GCTGAGAGCGCTACAGCGAACAGCTCTTTTCTACACATAAAGTGATAGCAGACAACTGTA\n") - fastaFileHandler.write("TGTGTGCACACGTGTGCTCATGCATTGTAAATTTGACAAAATATGCCCTTCACCTTCAAA\n") - fastaFileHandler.write(">DmelC:hr4_Blas-ter_Piler_1.0_Map_9\n") - fastaFileHandler.write("AGTTTAAAAACCAAAGACACTAGAATAACAAGATGCGTAACGGCCATACATTGGTTTGGC\n") - fastaFileHandler.write("ACTATGCAGCCACTTTTTTGGTGACGGCCAAAATTACTCTCTTTCCGCTCACTCCCGCTG\n") - fastaFileHandler.write("AGAGCGTAAGAAATCTAAAAATATAATTTGCTTGCTTGTGTGAGTAAAAACAAGAGACGA\n") - fastaFileHandler.write("GAACGCGTATAAGTGTGCGTGTTGTGCTAGAAGACGATTTTCGGGACCGAAATCAATTCT\n") - fastaFileHandler.write("GATCGAAGAAACGAATTTACATGGTACATATTAGGGTAGTTTTTGCCAATTTCCTAGCAA\n") - fastaFileHandler.close() - - def _writeFastaFile_with_pipe(self, fastaFileName): - fastaFileHandler = open(fastaFileName, "w") - fastaFileHandler.write(">DmelChr4_Blaster_Piler_0.0_Map_3\n") - fastaFileHandler.write("ACCAAAGACACTAGAATAACAAGATGCGTAACGCCATACGATTTTTTGGCACACTATTTT\n") - fastaFileHandler.write("TTCGCCGTGGCTCTAGAGGTGGCTCCAGGCTCTCTCGAATTTTTGTTAGAGAGCGAGAGA\n") - fastaFileHandler.write("GCTGAGAGCGCTACAGCGAACAGCTCTTTTCTACACATAAAGTGATAGCAGACAACTGTA\n") - fastaFileHandler.write("TGTGTGCACACGTGTGCTCATGCATTGTAAATTTGACAAAATATGCCCTTCACCTTCAAA\n") - fastaFileHandler.write(">DmelC|hr4_Blas-ter_Piler_1.0_Map_9\n") - fastaFileHandler.write("AGTTTAAAAACCAAAGACACTAGAATAACAAGATGCGTAACGGCCATACATTGGTTTGGC\n") - fastaFileHandler.write("ACTATGCAGCCACTTTTTTGGTGACGGCCAAAATTACTCTCTTTCCGCTCACTCCCGCTG\n") - fastaFileHandler.write("AGAGCGTAAGAAATCTAAAAATATAATTTGCTTGCTTGTGTGAGTAAAAACAAGAGACGA\n") - fastaFileHandler.write("GAACGCGTATAAGTGTGCGTGTTGTGCTAGAAGACGATTTTCGGGACCGAAATCAATTCT\n") - fastaFileHandler.write("GATCGAAGAAACGAATTTACATGGTACATATTAGGGTAGTTTTTGCCAATTTCCTAGCAA\n") - fastaFileHandler.close() - - def _writeFastaFile_with_equal(self, fastaFileName): - fastaFileHandler = open(fastaFileName, "w") - fastaFileHandler.write(">DmelChr4_Blaster_Piler_0.0_Map_3\n") - fastaFileHandler.write("ACCAAAGACACTAGAATAACAAGATGCGTAACGCCATACGATTTTTTGGCACACTATTTT\n") - fastaFileHandler.write("TTCGCCGTGGCTCTAGAGGTGGCTCCAGGCTCTCTCGAATTTTTGTTAGAGAGCGAGAGA\n") - fastaFileHandler.write("GCTGAGAGCGCTACAGCGAACAGCTCTTTTCTACACATAAAGTGATAGCAGACAACTGTA\n") - fastaFileHandler.write("TGTGTGCACACGTGTGCTCATGCATTGTAAATTTGACAAAATATGCCCTTCACCTTCAAA\n") - fastaFileHandler.write(">DmelC:hr4_Blas=ter_Piler_1.0_Map_9\n") - fastaFileHandler.write("AGTTTAAAAACCAAAGACACTAGAATAACAAGATGCGTAACGGCCATACATTGGTTTGGC\n") - fastaFileHandler.write("ACTATGCAGCCACTTTTTTGGTGACGGCCAAAATTACTCTCTTTCCGCTCACTCCCGCTG\n") - fastaFileHandler.write("AGAGCGTAAGAAATCTAAAAATATAATTTGCTTGCTTGTGTGAGTAAAAACAAGAGACGA\n") - fastaFileHandler.write("GAACGCGTATAAGTGTGCGTGTTGTGCTAGAAGACGATTTTCGGGACCGAAATCAATTCT\n") - fastaFileHandler.write("GATCGAAGAAACGAATTTACATGGTACATATTAGGGTAGTTTTTGCCAATTTCCTAGCAA\n") - fastaFileHandler.close() - - def _writeChangeLogFile(self, changeLogFileName ): - changeLogFileHandler = open(changeLogFileName, "w") - changeLogFileHandler.write("ChangeLog of REPET\n") - changeLogFileHandler.write("\n") - changeLogFileHandler.write("\n") - changeLogFileHandler.write("\n") - changeLogFileHandler.write("REPET release 1.3.6\n") - changeLogFileHandler.write("(release date XX/XX/2010)\n") - changeLogFileHandler.write("\n") - changeLogFileHandler.close() - - def _writeConfigFile(self, lineVersion): - configFileHandler = open(self.configFileName, "w") - configFileHandler.write("[repet_env]\n") - configFileHandler.write(lineVersion) - configFileHandler.write("repet_host: <your_MySQL_host>\n") - configFileHandler.close() - - -test_suite = unittest.TestSuite() -test_suite.addTest( unittest.makeSuite( Test_CheckerUtils ) ) -if __name__ == "__main__": - unittest.TextTestRunner(verbosity=2).run( test_suite )
--- a/commons/core/checker/test/Test_ConfigChecker.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,569 +0,0 @@ -from commons.core.checker.ConfigChecker import ConfigChecker -from commons.core.checker.ConfigChecker import ConfigRules -from commons.core.checker.RepetException import RepetException -import os -import unittest - -class Test_ConfigChecker(unittest.TestCase): - - def setUp(self): - self._configFileName = "testConfigChecker.cfg" - self._iMock = MockConfig() - - def test_checkIfExistsConfigFile_file_exist(self): - f=open(self._configFileName, "w") - f.close() - - doesFileExists = True - iConfigRules = ConfigRules() - try: - iConfigChecker = ConfigChecker(self._configFileName, iConfigRules) - iConfigChecker.checkIfExistsConfigFile() - except RepetException: - doesFileExists = False - os.remove(self._configFileName) - self.assertTrue(doesFileExists) - - def test_checkIfExistsConfigFile_file_not_exist(self): - iConfigRules = ConfigRules() - expMsg ="CONFIG FILE not found - '%s'" %self._configFileName - doesFileExists = True - try: - iConfigChecker = ConfigChecker(self._configFileName, iConfigRules) - iConfigChecker.checkIfExistsConfigFile() - except RepetException, re: - doesFileExists = False - self.assertFalse(doesFileExists) - self.assertEqual(expMsg, re.getMessage()) - - def test_readConfigFile(self): - self._iMock.write_config(self._configFileName) - iConfigRules = ConfigRules() - expDictRawConfigValues = {"dir_name" : {"work_dir":"toto"}, - "organism" : {"abbreviation":"T.aestivum", - "genus":"triticum", - "species":"aestivum", - "common_name":"wheat", - "comment":""}, - 'analysis1': {'description': '', - 'gff_name': 'BLASTX.gff2', - 'name': 'BLASTXWheat2', - 'program': 'BLASTX2', - 'programversion': '3.32', - 'sourcename': 'dummyDesc_BLASTX2'} - } - isNoExceptionRaised = True - try: - iConfigChecker = ConfigChecker(self._configFileName, iConfigRules) - iConfig = iConfigChecker.readConfigFile() - iConfigChecker.setRawConfig(iConfig) - obsDictRawConfigValues = iConfigChecker._iRawConfig.dOptionsValues4Sections - except RepetException: - isNoExceptionRaised = False - os.remove(self._configFileName) - self.assertTrue(isNoExceptionRaised) - self.assertEquals(obsDictRawConfigValues, expDictRawConfigValues) - - def test_readConfigFile_section_define_twice(self): - self._iMock.write_case_section_define_twice(self._configFileName) - iConfigRules = ConfigRules() - expMsg = "Duplicate section exist in config file %s" %self._configFileName - expDictRawConfigValues = {"dir_name": {"work_dir":"toto"}, - "analysis1" : {"name": "BLASTXWheat2", - "program" : "BLASTX2", - "programversion" : "3.32", - "sourcename" :"dummyDesc_BLASTX2", - "description" : "", - "gff_name" :"BLASTX.gff2"} - } - doesNoExceptionRaised = True - try: - iConfigChecker = ConfigChecker(self._configFileName, iConfigRules) - iConfig = iConfigChecker.readConfigFile() - iConfigChecker.setRawConfig(iConfig) - obsDictRawConfigValues = iConfigChecker._iRawConfig.dOptionsValues4Sections - except RepetException, re: - doesNoExceptionRaised = False - os.remove(self._configFileName) -# self.assertFalse(doesNoExceptionRaised) -# self.assertEqual(expMsg, re.getMessage()) -# - self.assertTrue(doesNoExceptionRaised) - self.assertEquals(obsDictRawConfigValues, expDictRawConfigValues) - - def test_readConfigFile_option_define_twice(self): - self._iMock.write_case_option_define_twice(self._configFileName) - iConfigRules = ConfigRules() - doesNoExceptionRaised = True - expDictRawConfigValues = {"dir_name": {"work_dir":"toto"}, - "analysis1" : {"name": "BLASTXWheat", - "program" : "BLASTX2", - "programversion" : "3.3", - "sourcename" :"dummyDesc_BLASTX", - "description" : "", - "gff_name" :"BLASTX.gff"} - } - try: - iConfigChecker = ConfigChecker(self._configFileName, iConfigRules) - iConfig = iConfigChecker.readConfigFile() - iConfigChecker.setRawConfig(iConfig) - obsDictRawConfigValues = iConfigChecker._iRawConfig.dOptionsValues4Sections - except RepetException, re: - doesNoExceptionRaised = False - os.remove(self._configFileName) -## self.assertFalse(doesNoExceptionRaised) -## self.assertEqual(expMsg, re.getMessage()) - - self.assertTrue(doesNoExceptionRaised) - self.assertEquals(obsDictRawConfigValues, expDictRawConfigValues) - - def test_checkMandatorySections(self): - self._iMock.write_config(self._configFileName) - iConfigRules = ConfigRules() - iConfigRules.addRuleSection(section="dir_name", mandatory=True) - iConfigRules.addRuleSection(section="organism", mandatory=True) - iConfigChecker = ConfigChecker(self._configFileName, iConfigRules) - iConfig = iConfigChecker.readConfigFile() - iConfigChecker.setRawConfig(iConfig) - iConfigChecker.extendConfigRulesWithPatternRules() - areAllMandatorySectionsFound = True - try: - iConfigChecker.checkMandatorySections() - except RepetException: - areAllMandatorySectionsFound = False - os.remove(self._configFileName) - self.assertTrue(areAllMandatorySectionsFound) - - def test_checkMandatorySections_one_missing (self): - self._iMock.write_config(self._configFileName) - iConfigRules = ConfigRules() - iConfigRules.addRuleSection(section="dir_name", mandatory=True) - iConfigRules.addRuleSection(section="target", mandatory=True) - iConfigChecker = ConfigChecker(self._configFileName, iConfigRules) - iConfig = iConfigChecker.readConfigFile() - iConfigChecker.setRawConfig(iConfig) - iConfigChecker.extendConfigRulesWithPatternRules() - expMsg = "Error in configuration file %s, following sections are missing:\n - target\n"% self._configFileName - areAllMandatorySectionsFound = True - try: - iConfigChecker.checkMandatorySections() - except RepetException, re: - areAllMandatorySectionsFound = False - os.remove(self._configFileName) - self.assertFalse(areAllMandatorySectionsFound) - self.assertEquals(expMsg, re.getMessage()) - - def test_checkMandatorySections_mandatory_section_with_pattern_section_is_missing (self): - self._iMock.write_config(self._configFileName) - iConfigRules = ConfigRules() - iConfigRules.addRuleSection(section="dir_name", mandatory=True) - iConfigRules.addRuleSection(section="mandatorySection", mandatory=True, isPattern = True) - iConfigChecker = ConfigChecker(self._configFileName, iConfigRules) - iConfig = iConfigChecker.readConfigFile() - iConfigChecker.setRawConfig(iConfig) - iConfigChecker.extendConfigRulesWithPatternRules() - expMsg = "Error in configuration file %s, following sections are missing:\n - mandatorySection\n"% self._configFileName - areAllMandatorySectionsFound = True - try: - iConfigChecker.checkMandatorySections() - except RepetException, re: - areAllMandatorySectionsFound = False - os.remove(self._configFileName) - self.assertFalse(areAllMandatorySectionsFound) - self.assertEquals(expMsg, re.getMessage()) - - def test_checkMandatorySections_mandatory_section_is_pattern (self): - self._iMock.write_case_pattern_rule(self._configFileName) - iConfigRules = ConfigRules() - iConfigRules.addRuleSection(section="dir_name", mandatory=True) - iConfigRules.addRuleSection(section="analysis[0-9]*", mandatory=True, isPattern = True) - iConfigChecker = ConfigChecker(self._configFileName, iConfigRules) - iConfig = iConfigChecker.readConfigFile() - iConfigChecker.setRawConfig(iConfig) - iConfigChecker.extendConfigRulesWithPatternRules() - areAllMandatorySectionsFound = True - try: - iConfigChecker.checkMandatorySections() - except RepetException: - areAllMandatorySectionsFound = False - os.remove(self._configFileName) - self.assertTrue(areAllMandatorySectionsFound) - - def test_checkMandatoryOptions_in_mandatory_section (self): - self._iMock.write_config(self._configFileName) - iConfigRules = ConfigRules() - iConfigRules.addRuleSection(section="organism", mandatory=True) - iConfigRules.addRuleOption(section="organism", option ="genus", mandatory=True) - iConfigChecker = ConfigChecker(self._configFileName, iConfigRules) - iConfig = iConfigChecker.readConfigFile() - iConfigChecker.setRawConfig(iConfig) - iConfigChecker.extendConfigRulesWithPatternRules() - areAllMandatoryOptionsFound = True - try: - iConfigChecker.checkMandatoryOptions() - except RepetException: - areAllMandatoryOptionsFound = False - os.remove(self._configFileName) - self.assertTrue(areAllMandatoryOptionsFound) - - def test_checkMandatoryOptions_in_mandatory_section_option_is_missing (self): - self._iMock.write_config(self._configFileName) - iConfigRules = ConfigRules() - iConfigRules.addRuleSection(section="organism", mandatory=True) - iConfigRules.addRuleOption(section="organism", option ="MissingOption", mandatory=True) - iConfigChecker = ConfigChecker(self._configFileName, iConfigRules) - iConfig = iConfigChecker.readConfigFile() - iConfigChecker.setRawConfig(iConfig) - iConfigChecker.extendConfigRulesWithPatternRules() - expMsg = "Error in configuration file %s, following options are missing: \n - [organism]: MissingOption\n" % self._configFileName - areAllMandatoryOptionsFound = True - try: - iConfigChecker.checkMandatoryOptions() - except RepetException, re: - areAllMandatoryOptionsFound = False - os.remove(self._configFileName) - self.assertFalse(areAllMandatoryOptionsFound) - self.assertEquals(expMsg, re.getMessage()) - - def test_checkMandatoryOptions_in_non_mandatory_section_and_section_and_option_exist (self): - self._iMock.write_config(self._configFileName) - iConfigRules = ConfigRules() - iConfigRules.addRuleOption(section="organism", option ="genus", mandatory=True) - iConfigChecker = ConfigChecker(self._configFileName, iConfigRules) - iConfig = iConfigChecker.readConfigFile() - iConfigChecker.setRawConfig(iConfig) - iConfigChecker.extendConfigRulesWithPatternRules() - areAllMandatoryOptionsFound = True - try: - iConfigChecker.checkMandatoryOptions() - except RepetException: - areAllMandatoryOptionsFound = False - os.remove(self._configFileName) - self.assertTrue(areAllMandatoryOptionsFound) - - def test_checkMandatoryOptions_in_non_mandatory_section_and_section_exist_option_is_missing (self): - self._iMock.write_config(self._configFileName) - iConfigRules = ConfigRules() - iConfigRules.addRuleOption(section="organism", option ="MissingOption", mandatory=True) - iConfigChecker = ConfigChecker(self._configFileName, iConfigRules) - iConfig = iConfigChecker.readConfigFile() - iConfigChecker.setRawConfig(iConfig) - iConfigChecker.extendConfigRulesWithPatternRules() - expMsg = "Error in configuration file %s, following options are missing: \n - [organism]: MissingOption\n" % self._configFileName - areAllMandatoryOptionsFound = True - try: - iConfigChecker.checkMandatoryOptions() - except RepetException, re: - areAllMandatoryOptionsFound = False - os.remove(self._configFileName) - self.assertFalse(areAllMandatoryOptionsFound) - self.assertEquals(expMsg,re.getMessage()) - - def test_checkMandatoryOptions_in_non_mandatory_section_and_section_does_not_exist (self): - self._iMock.write_config(self._configFileName) - iConfigRules = ConfigRules() - iConfigRules.addRuleOption(section="NonExistingAndNonMandatorySection", option ="genus", mandatory=True) - iConfigChecker = ConfigChecker(self._configFileName, iConfigRules) - iConfig = iConfigChecker.readConfigFile() - iConfigChecker.setRawConfig(iConfig) - iConfigChecker.extendConfigRulesWithPatternRules() - areAllMandatoryOptionsFound = True - try: - iConfigChecker.checkMandatoryOptions() - except RepetException: - areAllMandatoryOptionsFound = False - os.remove(self._configFileName) - self.assertTrue(areAllMandatoryOptionsFound) - - def test_checkMandatoryOptions_with_Pattern_rules_in_section (self): - self._iMock.write_case_pattern_rule(self._configFileName) - iConfigRules = ConfigRules() - iConfigRules.addRuleSection(section="analysis", isPattern=True) - iConfigRules.addRuleOption(section="analysis", option ="name", mandatory=True) - iConfigChecker = ConfigChecker(self._configFileName, iConfigRules) - iConfig = iConfigChecker.readConfigFile() - iConfigChecker.setRawConfig(iConfig) - iConfigChecker.extendConfigRulesWithPatternRules() - areAllMandatoryOptionsFound = True - try: - iConfigChecker.checkMandatoryOptions() - except RepetException: - areAllMandatoryOptionsFound = False - os.remove(self._configFileName) - self.assertTrue(areAllMandatoryOptionsFound) - - def test_checkMandatoryOptions_with_pattern_rules_in_option_section_is_mandatory (self): - self._iMock.write_case_pattern_rule(self._configFileName) - iConfigRules = ConfigRules() - iConfigRules.addRuleSection(section="section_with_option_pattern", mandatory=True) - iConfigRules.addRuleOption(section="section_with_option_pattern", option ="option", isPattern= True, mandatory=True) - iConfigChecker = ConfigChecker(self._configFileName, iConfigRules) - iConfig = iConfigChecker.readConfigFile() - iConfigChecker.setRawConfig(iConfig) - iConfigChecker.extendConfigRulesWithPatternRules() - areAllMandatoryOptionsFound = True - try: - iConfigChecker.checkMandatoryOptions() - except RepetException: - areAllMandatoryOptionsFound = False - os.remove(self._configFileName) - self.assertTrue(areAllMandatoryOptionsFound) - - def test_checkMandatoryOptions_with_pattern_rules_in_option_in_mandatory_section_option_is_missing (self): - self._iMock.write_case_pattern_rule(self._configFileName) - iConfigRules = ConfigRules() - iConfigRules.addRuleSection(section="section_with_option_pattern", mandatory=True) - iConfigRules.addRuleOption(section="section_with_option_pattern", option ="MissingOption", isPattern= True, mandatory=True) - iConfigChecker = ConfigChecker(self._configFileName, iConfigRules) - iConfig = iConfigChecker.readConfigFile() - iConfigChecker.setRawConfig(iConfig) - iConfigChecker.extendConfigRulesWithPatternRules() - expMsg = "Error in configuration file %s, following options are missing: \n - [section_with_option_pattern]: MissingOption\n" % self._configFileName - areAllMandatoryOptionsFound = True - try: - iConfigChecker.checkMandatoryOptions() - except RepetException, re: - areAllMandatoryOptionsFound = False - os.remove(self._configFileName) - self.assertFalse(areAllMandatoryOptionsFound) - self.assertEquals(expMsg, re.getMessage()) - - def test_checkMandatoryOptions_with_pattern_rules_in_non_mandatory_section_and_section_and_option_exist (self): - self._iMock.write_case_pattern_rule(self._configFileName) - iConfigRules = ConfigRules() - iConfigRules.addRuleOption(section="section_with_option_pattern", option ="option", isPattern= True, mandatory=True) - iConfigChecker = ConfigChecker(self._configFileName, iConfigRules) - iConfig = iConfigChecker.readConfigFile() - iConfigChecker.setRawConfig(iConfig) - iConfigChecker.extendConfigRulesWithPatternRules() - areAllMandatoryOptionsFound = True - try: - iConfigChecker.checkMandatoryOptions() - except RepetException: - areAllMandatoryOptionsFound = False - os.remove(self._configFileName) - self.assertTrue(areAllMandatoryOptionsFound) - - def test_checkMandatoryOptions_with_pattern_rules_in_non_mandatory_section_and_section_exist_option_is_missing (self): - self._iMock.write_case_pattern_rule(self._configFileName) - iConfigRules = ConfigRules() - iConfigRules.addRuleOption(section="section_with_option_pattern", option ="MissingOption", isPattern= True, mandatory=True) - iConfigChecker = ConfigChecker(self._configFileName, iConfigRules) - iConfig = iConfigChecker.readConfigFile() - iConfigChecker.setRawConfig(iConfig) - iConfigChecker.extendConfigRulesWithPatternRules() - expMsg = "Error in configuration file %s, following options are missing: \n - [section_with_option_pattern]: MissingOption\n" % self._configFileName - areAllMandatoryOptionsFound = True - try: - iConfigChecker.checkMandatoryOptions() - except RepetException, re: - areAllMandatoryOptionsFound = False - os.remove(self._configFileName) - self.assertFalse(areAllMandatoryOptionsFound) - self.assertEquals(expMsg,re.getMessage()) - - def test_checkMandatoryOptions_with_pattern_rules_in_non_mandatory_section_and_section_does_not_exist (self): - self._iMock.write_case_pattern_rule(self._configFileName) - iConfigRules = ConfigRules() - iConfigRules.addRuleOption(section="non_mandatory_section", option ="MissingOption", isPattern= True, mandatory=True) - iConfigChecker = ConfigChecker(self._configFileName, iConfigRules) - iConfig = iConfigChecker.readConfigFile() - iConfigChecker.setRawConfig(iConfig) - iConfigChecker.extendConfigRulesWithPatternRules() - areAllMandatoryOptionsFound = True - try: - iConfigChecker.checkMandatoryOptions() - except RepetException: - areAllMandatoryOptionsFound = False - os.remove(self._configFileName) - self.assertTrue(areAllMandatoryOptionsFound) - - def test_checkMandatoryOptions_with_pattern_rules_for_both_section_and_option (self): - self._iMock.write_case_pattern_rule(self._configFileName) - iConfigRules = ConfigRules() - iConfigRules.addRuleSection(section="section_with_option_pattern", isPattern=True) - iConfigRules.addRuleOption(section="section_with_option_pattern", option ="option", isPattern= True, mandatory=True) - iConfigChecker = ConfigChecker(self._configFileName, iConfigRules) - iConfig = iConfigChecker.readConfigFile() - iConfigChecker.setRawConfig(iConfig) - iConfigChecker.extendConfigRulesWithPatternRules() - areAllMandatoryOptionsFound = True - try: - iConfigChecker.checkMandatoryOptions() - except RepetException: - areAllMandatoryOptionsFound = False - os.remove(self._configFileName) - self.assertTrue(areAllMandatoryOptionsFound) - - def test_checkMandatoryOptions_case(self): - self._iMock.write_config_case(self._configFileName) - iConfigRules = ConfigRules() - iConfigRules.addRuleSection("dir_name", True) - iConfigRules.addRuleSection("organism", True) - iConfigRules.addRuleOption("organism", "min_SSR_coverage", True) - iConfigChecker = ConfigChecker(self._configFileName, iConfigRules) - iConfig = iConfigChecker.readConfigFile() - iConfigChecker.setRawConfig(iConfig) - iConfigChecker.extendConfigRulesWithPatternRules() - areAllMandatorySectionsFound = True - try: - iConfigChecker.checkMandatoryOptions() - except RepetException: - areAllMandatorySectionsFound = False - os.remove(self._configFileName) - self.assertTrue(areAllMandatorySectionsFound) - -#TODO Test de extendConfigRulesWithPatternRules et de applyRuleToRawValue -# option avec une liste de valeurs possibles dans un ensemble -# option type=string type="num" type="date"???. -#option define twice and options use in other - -class MockConfig (object): - - def write_config(self, configFileName): - configF = open(configFileName, "w" ) - configF.write( "[dir_name]\n") - configF.write( "work_dir : toto \n") - configF.write( "\n") - configF.write( "[organism]\n") - configF.write( "abbreviation: T.aestivum\n") - configF.write( "genus: triticum\n") - configF.write( "species: aestivum\n") - configF.write( "common_name: wheat\n") - configF.write( "comment: \n") - configF.write( "[analysis1]\n") - configF.write( "name: BLASTXWheat2\n") - configF.write( "program: BLASTX2\n") - configF.write( "programversion: 3.32\n") - configF.write( "sourcename: dummyDesc_BLASTX2\n") - configF.write( "description: \n") - configF.write( "gff_name: BLASTX.gff2\n") - configF.write( "\n") - configF.close() - - def write_case_section_define_twice(self, configFileName): - configF = open(configFileName, "w" ) - configF.write( "[dir_name]\n") - configF.write( "work_dir : toto \n") - configF.write( "\n") - configF.write( "[analysis1]\n") - configF.write( "name: BLASTXWheat\n") - configF.write( "program: BLASTX\n") - configF.write( "programversion: 3.3\n") - configF.write( "sourcename: dummyDesc_BLASTX\n") - configF.write( "description: \n") - configF.write( "gff_name: BLASTX.gff\n") - configF.write( "\n") - configF.write( "[analysis1]\n") - configF.write( "name: BLASTXWheat2\n") - configF.write( "program: BLASTX2\n") - configF.write( "programversion: 3.32\n") - configF.write( "sourcename: dummyDesc_BLASTX2\n") - configF.write( "description: \n") - configF.write( "gff_name: BLASTX.gff2\n") - configF.write( "\n") - configF.close() - - def write_case_option_define_twice(self, configFileName): - configF = open(configFileName, "w" ) - configF.write( "[dir_name]\n") - configF.write( "work_dir : toto \n") - configF.write( "\n") - configF.write( "[analysis1]\n") - configF.write( "name: BLASTXWheat\n") - configF.write( "program: BLASTX\n") - configF.write( "programversion: 3.3\n") - configF.write( "sourcename: dummyDesc_BLASTX\n") - configF.write( "program: BLASTX2\n") - configF.write( "description: \n") - configF.write( "gff_name: BLASTX.gff\n") - configF.write( "\n") - configF.write( "\n") - configF.close() - - #configuration file with section with option depends on presence of other options - def write_with_one_option_depends_of_an_other_one(self, configFileName ): - configF = open(configFileName, "w" ) - configF.write( "[dir_name]\n") - configF.write( "work_dir : toto\n") - configF.write( "\n") - configF.write( "[organism]\n") - configF.write( "abbreviation: T.aestivum\n") - configF.write( "genus: Triticum\n") - configF.write( "species: aestivum\n") - configF.write( "common_name: wheat\n") - configF.write( "comment: \n") - configF.write( "\n") - configF.write( "[analysis1]\n") - configF.write( "name: BLASTXWheat\n") - configF.write( "program: BLASTX\n") - configF.write( "programversion: 3.3\n") - configF.write( "sourcename: src_BLASTX\n") - configF.write( "description: \n") - configF.write( "gff_name: BLASTX.gff\n") - configF.write( "\n") - configF.write( "[analysis2]\n") - configF.write( "name: GMHMMWheat\n") - configF.write( "program: GMHMM\n") - configF.write( "programversion: 4.3\n") - configF.write( "sourcename: src_GMHMM\n") - configF.write( "description: \n") - configF.write( "gff_name: GMHMM.gff\n") - configF.write( "\n") - configF.write( "[target]\n") - configF.write( "target_used: yes\n") - configF.write( "target_used_list: target.lst\n") - configF.close() - - def write_case_pattern_rule(self, configFileName ): - configF = open(configFileName, "w" ) - configF.write( "[dir_name]\n") - configF.write( "work_dir : toto\n" ) - configF.write( "\n") - configF.write( "[organism]\n") - configF.write( "abbreviation: T.aestivum\n") - configF.write( "genus: Triticum\n") - configF.write( "species: aestivum\n") - configF.write( "common_name: wheat\n") - configF.write( "comment: \n") - configF.write( "\n") - configF.write( "[analysis1]\n") - configF.write( "name: BLASTXWheat\n") - configF.write( "program: BLASTX\n") - configF.write( "programversion: 3.3\n") - configF.write( "sourcename: src_BLASTX\n") - configF.write( "description: \n") - configF.write( "gff_name: BLASTX.gff\n") - configF.write( "\n") - configF.write( "[analysis2]\n") - configF.write( "name: GMHMMWheat\n") - configF.write( "program: GMHMM\n") - configF.write( "programversion: 4.3\n") - configF.write( "sourcename: src_GMHMM\n") - configF.write( "description: \n") - configF.write( "gff_name: GMHMM.gff\n") - configF.write( "\n") - configF.write( "[target]\n") - configF.write( "target_used: yes\n") - configF.write( "target_used_list: target.lst\n") - configF.write( "\n") - configF.write( "[section_with_option_pattern]\n") - configF.write( "option1: value1\n") - configF.write( "option2: value2\n") - configF.write( "[second_section_with_option_pattern]\n") - configF.write( "option1: value1\n") - configF.write( "option2: value2\n") - configF.close() - - def write_config_case(self, configFileName): - configF = open(configFileName, "w" ) - configF.write( "[dir_name]\n") - configF.write( "work_dir : toto \n") - configF.write( "\n") - configF.write( "[organism]\n") - configF.write( "min_SSR_coverage: 0.50\n") - configF.write( "\n") - configF.close() - -if __name__ == "__main__": - unittest.main() \ No newline at end of file
--- a/commons/core/checker/test/Test_ConfigValue.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,217 +0,0 @@ -import unittest -from commons.core.checker.ConfigValue import ConfigValue - -class Test_ConfigValue(unittest.TestCase): - - def setUp(self): - self._iConfigValue = ConfigValue() - - def test__eq__True(self): - self._iConfigValue.dOptionsValues4Sections = { - "dir_name" : {"work_dir":"toto"}, - "organism" : {"abbreviation":"T.aestivum", - "genus":"triticum", - "species":"aestivum", - "common_name":"wheat", - "comment":""} - } - iConfigValue1 = ConfigValue() - iConfigValue1.dOptionsValues4Sections = { - "dir_name" : {"work_dir":"toto"}, - "organism" : {"abbreviation":"T.aestivum", - "genus":"triticum", - "species":"aestivum", - "common_name":"wheat", - "comment":""} - } - - self.assertEqual(self._iConfigValue, iConfigValue1) - - def test__eq__False_not_same_section(self): - self._iConfigValue.dOptionsValues4Sections = { - "dir_name" : {"work_dir":"toto"}, - "organisms" : {"abbreviation":"T.aestivum", - "genus":"triticum", - "species":"aestivum", - "common_name":"wheat", - "comment":""} - } - iConfigValue1 = ConfigValue() - iConfigValue1.dOptionsValues4Sections = { - "dir_name" : {"work_dir":"toto"}, - "organism" : {"abbreviation":"T.aestivum", - "genus":"triticum", - "species":"aestivum", - "common_name":"wheat", - "comment":""} - } - - self.assertNotEqual(self._iConfigValue, iConfigValue1) - - - def test__eq__False_not_same_option(self): - self._iConfigValue.dOptionsValues4Sections = { - "dir_name" : {"work_dir":"toto"}, - "organism" : {"abbreviation":"T.aestivum", - "family":"triticum", - "species":"aestivum", - "common_name":"wheat", - "comment":""} - } - iConfigValue1 = ConfigValue() - iConfigValue1.dOptionsValues4Sections = { - "dir_name" : {"work_dir":"toto"}, - "organism" : {"abbreviation":"T.aestivum", - "genus":"triticum", - "species":"aestivum", - "common_name":"wheat", - "comment":""} - } - - self.assertNotEqual(self._iConfigValue, iConfigValue1) - - def test__eq__False_not_same_value(self): - self._iConfigValue.dOptionsValues4Sections = { - "dir_name" : {"work_dir":"toto"}, - "organism" : {"abbreviation":"T.aestivum", - "genus":"vitis", - "species":"aestivum", - "common_name":"wheat", - "comment":""} - } - iConfigValue1 = ConfigValue() - iConfigValue1.dOptionsValues4Sections = { - "dir_name" : {"work_dir":"toto"}, - "organism" : {"abbreviation":"T.aestivum", - "genus":"triticum", - "species":"aestivum", - "common_name":"wheat", - "comment":""} - } - - self.assertNotEqual(self._iConfigValue, iConfigValue1) - - def test_has_section_true(self): - self._iConfigValue.dOptionsValues4Sections = { - "dir_name" : {"work_dir":"toto"}, - "organism" : {"abbreviation":"T.aestivum", - "genus":"triticum", - "species":"aestivum", - "common_name":"wheat", - "comment":""} - } - - obsSectionExist = self._iConfigValue.has_section("organism") - self.assertTrue(obsSectionExist) - - def test_has_section_false(self): - self._iConfigValue.dOptionsValues4Sections = { - "dir_name" : {"work_dir":"toto"}, - "organism" : {"abbreviation":"T.aestivum", - "genus":"triticum", - "species":"aestivum", - "common_name":"wheat", - "comment":""} - } - - obsSectionExist = self._iConfigValue.has_section("toto") - self.assertFalse(obsSectionExist) - - def test_has_option_true(self): - self._iConfigValue.dOptionsValues4Sections = { - "dir_name" : {"work_dir":"toto"}, - "organism" : {"abbreviation":"T.aestivum", - "genus":"triticum", - "species":"aestivum", - "common_name":"wheat", - "comment":""} - } - - obsOptionExist = self._iConfigValue.has_option("organism","genus") - self.assertTrue(obsOptionExist) - - def test_has_option_false(self): - self._iConfigValue.dOptionsValues4Sections = { - "dir_name" : {"work_dir":"toto"}, - "organism" : {"abbreviation":"T.aestivum", - "genus":"triticum", - "species":"aestivum", - "common_name":"wheat", - "comment":""} - } - - obsOptionExist = self._iConfigValue.has_option("organism","toto") - self.assertFalse(obsOptionExist) - obsOptionExist = self._iConfigValue.has_option("toto","genus") - self.assertFalse(obsOptionExist) - - def test_sections(self): - self._iConfigValue.dOptionsValues4Sections = { - "dir_name" : {"work_dir":"toto"}, - "organism" : {"abbreviation":"T.aestivum", - "genus":"triticum", - "species":"aestivum", - "common_name":"wheat", - "comment":""} - } - expListSections = ["dir_name", "organism"] - obsListSections = self._iConfigValue.sections() - self.assertEquals(expListSections, obsListSections) - - def test_sections_empty_config(self): - self._iConfigValue.dOptionsValues4Sections = {} - expListSections = [] - obsListSections = self._iConfigValue.sections() - self.assertEquals(expListSections, obsListSections) - - def test_options(self): - self._iConfigValue.dOptionsValues4Sections = { - "dir_name" : {"work_dir":"toto"}, - "organism" : {"abbreviation":"T.aestivum", - "genus":"triticum", - "species":"aestivum", - "common_name":"wheat", - "comment":""} - } - expListOptions = ['abbreviation', 'common_name', 'genus', 'species', 'comment'] - obsListOptions = self._iConfigValue.options("organism") - self.assertEquals(expListOptions, obsListOptions) - - expListOptions = ["work_dir"] - obsListOptions = self._iConfigValue.options("dir_name") - self.assertEquals(expListOptions, obsListOptions) - - def test_options_empty_config(self): - self._iConfigValue.dOptionsValues4Sections = {} - expListOptions = [] - obsListOptions = self._iConfigValue.options("toto") - self.assertEquals(expListOptions, obsListOptions) - - def test_set(self): - self._iConfigValue.dOptionsValues4Sections = {} - expDictOptionsValue = {"dir_name" : {"work_dir":"toto"}} - self._iConfigValue.set("dir_name", "work_dir", "toto") - obsDictOptionsValue = self._iConfigValue.dOptionsValues4Sections - self.assertEquals(expDictOptionsValue, obsDictOptionsValue) - - def test_get(self): - self._iConfigValue.dOptionsValues4Sections = { - "dir_name" : {"work_dir":"toto"}, - "organism" : {"abbreviation":"T.aestivum", - "genus":"triticum", - "species":"aestivum", - "common_name":"wheat", - "comment":""} - } - expValue = "aestivum" - obsValue = self._iConfigValue.get("organism", "species") - self.assertEquals(expValue, obsValue) - expValue = None - obsValue = self._iConfigValue.get("toto", "species") - self.assertEquals(expValue, obsValue) - expValue = None - obsValue = self._iConfigValue.get("organism", "dummyopt") - self.assertEquals(expValue, obsValue) - -if __name__ == "__main__": - unittest.main() \ No newline at end of file
--- a/commons/core/checker/test/Test_F_ConfigChecker.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,214 +0,0 @@ -from commons.core.checker.ConfigChecker import ConfigChecker -from commons.core.checker.ConfigChecker import ConfigRules -from commons.core.checker.ConfigValue import ConfigValue -from commons.core.checker.RepetException import RepetException -import unittest -import os - -class Test_F_ConfigChecker(unittest.TestCase): - - #TODO: AJouter test (wrong type, etc..) - def setUp(self): - self._configFileName = "test_conf_checker" - - def tearDown(self): - os.remove(self._configFileName) - - def test_run(self): - iMock = MockConfig() - iMock.write_config(self._configFileName) - - iConfigRules = ConfigRules() - iConfigRules.addRuleSection(section="dir_name", mandatory=True) - iConfigRules.addRuleOption(section="dir_name", option ="work_dir", mandatory=True) - iConfigRules.addRuleSection(section="organism", mandatory=True) - iConfigRules.addRuleOption(section="organism", option ="abbreviation", mandatory=True) - iConfigRules.addRuleOption(section="organism", option ="genus", mandatory=True) - iConfigRules.addRuleOption(section="organism", option ="species", mandatory=True) - iConfigRules.addRuleOption(section="organism", option ="common_name", mandatory=True) - iConfigRules.addRuleOption(section="organism", option ="comment") - iConfigRules.addRuleSection(section="analysis", mandatory=True, isPattern=True) - iConfigRules.addRuleOption(section="analysis", option ="name", mandatory=True) - iConfigRules.addRuleOption(section="analysis", option ="program", mandatory=True) - iConfigRules.addRuleOption(section="analysis", option ="sourcename", mandatory=True) - iConfigRules.addRuleOption(section="analysis", option ="description") - iConfigRules.addRuleOption(section="analysis", option ="gff_name") - - iConfigChecker = ConfigChecker(self._configFileName,iConfigRules) - - obsValidatedConfig = iConfigChecker.getConfig() - - expValidatedConfig = ConfigValue() - d = {"dir_name" : {"work_dir":"toto"}, - "organism" : {"abbreviation":"T.aestivum", - "genus":"triticum", - "species":"aestivum", - "common_name":"wheat", - "comment":""}, - 'analysis1': {'description': '', - 'gff_name': 'BLASTX.gff2', - 'name': 'BLASTXWheat2', - 'program': 'BLASTX2', - 'programversion': '3.32', - 'sourcename': 'dummyDesc_BLASTX2'} - } - expValidatedConfig.setdOptionsValues4Sections(d) - - self.assertEquals(expValidatedConfig, obsValidatedConfig) - - - def test_run_exception_section_missing(self): - iMock = MockConfig() - iMock.write_config_section_missing(self._configFileName) - - iConfigRules = ConfigRules() - iConfigRules.addRuleSection(section="dir_name", mandatory=True) - iConfigRules.addRuleOption(section="dir_name", option ="work_dir", mandatory=True) - iConfigRules.addRuleSection(section="organism", mandatory=True) - iConfigRules.addRuleOption(section="organism", option ="abbreviation", mandatory=True) - iConfigRules.addRuleOption(section="organism", option ="genus", mandatory=True) - iConfigRules.addRuleOption(section="organism", option ="species", mandatory=True) - iConfigRules.addRuleOption(section="organism", option ="common_name", mandatory=True) - iConfigRules.addRuleOption(section="organism", option ="comment") - iConfigRules.addRuleSection(section="analysis", mandatory=True, isPattern=True) - iConfigRules.addRuleOption(section="analysis", option ="name", mandatory=True) - iConfigRules.addRuleOption(section="analysis", option ="program", mandatory=True) - iConfigRules.addRuleOption(section="analysis", option ="sourcename", mandatory=True) - iConfigRules.addRuleOption(section="analysis", option ="description") - iConfigRules.addRuleOption(section="analysis", option ="gff_name") - - iConfigChecker = ConfigChecker(self._configFileName,iConfigRules) - - expMessage = "Error in configuration file %s, following sections are missing:\n - organism\n"% self._configFileName - - try : - obsValidatedConfig = iConfigChecker.getConfig() - except RepetException, e: - obsMessage = e.getMessage() - - self.assertEquals(expMessage, obsMessage) - - - def test_run_exception_section_pattern_false(self): - iMock = MockConfig() - iMock.write_config(self._configFileName) - - iConfigRules = ConfigRules() - iConfigRules.addRuleSection(section="dir_name", mandatory=True) - iConfigRules.addRuleOption(section="dir_name", option ="work_dir", mandatory=True) - iConfigRules.addRuleSection(section="organism", mandatory=True) - iConfigRules.addRuleOption(section="organism", option ="abbreviation", mandatory=True) - iConfigRules.addRuleOption(section="organism", option ="genus", mandatory=True) - iConfigRules.addRuleOption(section="organism", option ="species", mandatory=True) - iConfigRules.addRuleOption(section="organism", option ="common_name", mandatory=True) - iConfigRules.addRuleOption(section="organism", option ="comment") - iConfigRules.addRuleSection(section="analysis", mandatory=True) - iConfigRules.addRuleOption(section="analysis", option ="name", mandatory=True) - iConfigRules.addRuleOption(section="analysis", option ="program", mandatory=True) - iConfigRules.addRuleOption(section="analysis", option ="sourcename", mandatory=True) - iConfigRules.addRuleOption(section="analysis", option ="description") - iConfigRules.addRuleOption(section="analysis", option ="gff_name") - - iConfigChecker = ConfigChecker(self._configFileName,iConfigRules) - - expMessage = "Error in configuration file %s, following sections are missing:\n - analysis\n"% self._configFileName - - try : - obsValidatedConfig = iConfigChecker.getConfig() - except RepetException, e: - obsMessage = e.getMessage() - - self.assertEquals(expMessage, obsMessage) - - - def test_run_exception_option_missing(self): - iMock = MockConfig() - iMock.write_config_option_missing(self._configFileName) - - iConfigRules = ConfigRules() - iConfigRules.addRuleSection(section="dir_name", mandatory=True) - iConfigRules.addRuleOption(section="dir_name", option ="work_dir", mandatory=True) - iConfigRules.addRuleSection(section="organism", mandatory=True) - iConfigRules.addRuleOption(section="organism", option ="abbreviation", mandatory=True) - iConfigRules.addRuleOption(section="organism", option ="genus", mandatory=True) - iConfigRules.addRuleOption(section="organism", option ="species", mandatory=True) - iConfigRules.addRuleOption(section="organism", option ="common_name", mandatory=True) - iConfigRules.addRuleOption(section="organism", option ="comment") - iConfigRules.addRuleSection(section="analysis", mandatory=True, isPattern=True) - iConfigRules.addRuleOption(section="analysis", option ="name", mandatory=True) - iConfigRules.addRuleOption(section="analysis", option ="program", mandatory=True) - iConfigRules.addRuleOption(section="analysis", option ="sourcename", mandatory=True) - iConfigRules.addRuleOption(section="analysis", option ="description") - iConfigRules.addRuleOption(section="analysis", option ="gff_name") - - iConfigChecker = ConfigChecker(self._configFileName,iConfigRules) - - expMessage = "Error in configuration file %s, following options are missing: \n - [organism]: abbreviation\n"% self._configFileName - - try : - obsValidatedConfig = iConfigChecker.getConfig() - except RepetException, e: - obsMessage = e.getMessage() - - self.assertEquals(expMessage, obsMessage) - -class MockConfig (object): - - def write_config(self, configFileName): - configF = open(configFileName, "w" ) - configF.write( "[dir_name]\n") - configF.write( "work_dir : toto \n") - configF.write( "\n") - configF.write( "[organism]\n") - configF.write( "abbreviation: T.aestivum\n") - configF.write( "genus: triticum\n") - configF.write( "species: aestivum\n") - configF.write( "common_name: wheat\n") - configF.write( "comment: \n") - configF.write( "[analysis1]\n") - configF.write( "name: BLASTXWheat2\n") - configF.write( "program: BLASTX2\n") - configF.write( "programversion: 3.32\n") - configF.write( "sourcename: dummyDesc_BLASTX2\n") - configF.write( "description: \n") - configF.write( "gff_name: BLASTX.gff2\n") - configF.write( "\n") - configF.close() - - def write_config_section_missing(self, configFileName): - configF = open(configFileName, "w" ) - configF.write( "[dir_name]\n") - configF.write( "work_dir : toto \n") - configF.write( "\n") - configF.write( "[analysis1]\n") - configF.write( "name: BLASTXWheat2\n") - configF.write( "program: BLASTX2\n") - configF.write( "programversion: 3.32\n") - configF.write( "sourcename: dummyDesc_BLASTX2\n") - configF.write( "description: \n") - configF.write( "gff_name: BLASTX.gff2\n") - configF.write( "\n") - configF.close() - - def write_config_option_missing(self, configFileName): - configF = open(configFileName, "w" ) - configF.write( "[dir_name]\n") - configF.write( "work_dir : toto \n") - configF.write( "\n") - configF.write( "[organism]\n") - configF.write( "genus: triticum\n") - configF.write( "species: aestivum\n") - configF.write( "common_name: wheat\n") - configF.write( "comment: \n") - configF.write( "[analysis1]\n") - configF.write( "name: BLASTXWheat2\n") - configF.write( "program: BLASTX2\n") - configF.write( "programversion: 3.32\n") - configF.write( "sourcename: dummyDesc_BLASTX2\n") - configF.write( "description: \n") - configF.write( "gff_name: BLASTX.gff2\n") - configF.write( "\n") - configF.close() - -if __name__ == "__main__": - unittest.main() \ No newline at end of file
--- a/commons/core/checker/test/Test_OldConfigChecker.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,104 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - - -import unittest -import os -from commons.core.checker.OldConfigChecker import ConfigChecker -from commons.core.checker.ConfigException import ConfigException - -class Test_ConfigChecker( unittest.TestCase ): - - def setUp(self): - pass - - def tearDown(self): - pass - - - def testFileNotFound(self): - exceptionExpected = None - configChecker = ConfigChecker("",{}) - try : - configChecker.check("noExistsFile.cfg") - except ConfigException, ce: - exceptionExpected = ce - - self.assertTrue(exceptionExpected != None) - msg = exceptionExpected.messages[0] - self.assertTrue(msg.startswith("CONFIG FILE not found - ")) - - - def testNoSectionInConfigFile (self): - exceptionExpected = None - dummyFile = open("dummyFile.cfg", "w") - configChecker = ConfigChecker("dummySection",{}) - try : - configChecker.check("dummyFile.cfg") - except ConfigException, ce: - exceptionExpected = ce - - self.assertTrue(exceptionExpected != None) - msg = exceptionExpected.messages[0] - self.assertTrue(msg.startswith("[dummySection]" + " section not found - ")) - - os.remove("dummyFile.cfg") - - - def testNoOptionInConfigFile (self): - exceptionExpected = None - MockConfigFile("dummyConfig.cfg",{}) - configChecker = ConfigChecker("blaster_config",{"dummy":""}) - try : - configChecker.check("dummyConfig.cfg") - except ConfigException, ce: - exceptionExpected = ce - - self.assertTrue(exceptionExpected != None) - msg = exceptionExpected.messages[0] - self.assertTrue(msg.startswith("[blaster_config] - No option 'dummy' in section: 'blaster_config'")) - os.remove("dummyConfig.cfg") - - -class MockConfigFile: - - def __init__ (self, fileName, optionsDict): - self._fileName = fileName - config = open(fileName, "w"); - config.write("[blaster_config]\n") - for key in optionsDict.keys(): - config.write(key + ":" + optionsDict[key] + "\n") - config.close() - - -test_suite = unittest.TestSuite() -test_suite.addTest( unittest.makeSuite( Test_ConfigChecker ) ) -if __name__ == "__main__": - unittest.TextTestRunner(verbosity=2).run( test_suite )
--- a/commons/core/launcher/JobScriptTemplate.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,95 +0,0 @@ -#!/usr/bin/env python - -import os -import sys -import time -import shutil -from commons.core.checker.RepetException import RepetException -from commons.core.sql.TableJobAdaptator import TableJobAdaptator -from commons.core.sql.DbFactory import DbFactory -from commons.core.sql.Job import Job - -try: - newDir = None - print os.uname() - beginTime = time.time() - print 'beginTime=%f' % beginTime - print "work in dir '@@tmpDir@@'" - sys.stdout.flush() - if not os.path.exists( "@@tmpDir@@" ): - raise IOError("ERROR: temporary directory '@@tmpDir@@' doesn't exist") - - minFreeGigaInTmpDir = 1 - freeSpace = os.statvfs("@@tmpDir@@") - if ((freeSpace.f_bavail * freeSpace.f_frsize) / 1073741824.0 < minFreeGigaInTmpDir): - raise RepetException("ERROR: less than %iG of free space in '@@tmpDir@@'" % minFreeGigaInTmpDir) - - os.chdir("@@tmpDir@@") - newDir = "@@groupId@@_@@jobName@@_@@time@@" - if os.path.exists(newDir): - shutil.rmtree(newDir) - os.mkdir(newDir) - os.chdir(newDir) - - iJob = Job(jobname = "@@jobName@@", groupid = "@@groupId@@", launcherFile = "@@launcher@@", node = os.getenv("HOSTNAME")) - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "@@jobTableName@@") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "running") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - - @@cmdStart@@ - if log != 0: - raise RepetException("ERROR: job returned %i" % log) - else: - print "job finished successfully" - sys.stdout.flush() - @@cmdFinish@@ - - os.chdir("..") - shutil.rmtree(newDir) - - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "@@jobTableName@@") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "finished") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - - endTime = time.time() - print 'endTime=%f' % endTime - print 'executionTime=%f' % (endTime - beginTime) - print os.uname() - sys.stdout.flush() - -except IOError, e : - print e - iJob = Job(jobname = "@@jobName@@", groupid = "@@groupId@@", launcherFile = "@@launcher@@", node = os.getenv("HOSTNAME")) - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "@@jobTableName@@") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "error") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - sys.exit(1) - -except Exception, e : - print "tmpDir is : @@tmpDir@@" - print "cDir is : @@cDir@@" - print e - if newDir != None and os.path.exists("../%s" % newDir) and not os.path.exists("@@cDir@@/%s" % newDir): - os.chdir("..") - shutil.move(newDir, "@@cDir@@/%s" % newDir) - iJob = Job(jobname = "@@jobName@@", groupid = "@@groupId@@", launcherFile = "@@launcher@@", node = os.getenv("HOSTNAME")) - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "@@jobTableName@@") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "error") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - sys.exit(1)
--- a/commons/core/launcher/JobScriptTemplateLight.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,49 +0,0 @@ -#!/usr/bin/env python - -import os -import sys -import time -import shutil -from commons.core.checker.RepetException import RepetException -try: - newDir = None - print os.uname() - beginTime = time.time() - print 'beginTime=%f' % beginTime - print "work in dir '@@tmpDir@@'" - sys.stdout.flush() - if not os.path.exists( "@@tmpDir@@" ): - raise IOError("ERROR: temporary directory '@@tmpDir@@' doesn't exist") - - minFreeGigaInTmpDir = 1 - freeSpace = os.statvfs("@@tmpDir@@") - if ((freeSpace.f_bavail * freeSpace.f_frsize) / 1073741824.0 < minFreeGigaInTmpDir): - raise RepetException("ERROR: less than %iG of free space in '@@tmpDir@@'" % minFreeGigaInTmpDir) - - os.chdir("@@tmpDir@@") - newDir = "@@groupId@@_@@jobName@@_@@time@@" - if os.path.exists(newDir): - shutil.rmtree(newDir) - os.mkdir(newDir) - os.chdir(newDir) - - @@cmdStart@@ - if log != 0: - raise RepetException("ERROR: job returned %i" % log) - else: - print "job finished successfully" - sys.stdout.flush() - @@cmdFinish@@ - - os.chdir("..") - shutil.rmtree(newDir) - endTime = time.time() - print 'endTime=%f' % endTime - print 'executionTime=%f' % (endTime - beginTime) - print os.uname() - sys.stdout.flush() - -except IOError, e : - print e - sys.stdout.flush() - sys.exit(1) \ No newline at end of file
--- a/commons/core/launcher/JobScriptWithFilesCopyTemplate.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,109 +0,0 @@ -#!/usr/bin/env python - -import os -import sys -import time -import shutil -from commons.core.checker.RepetException import RepetException -from commons.core.sql.TableJobAdaptator import TableJobAdaptator -from commons.core.sql.DbFactory import DbFactory -from commons.core.sql.Job import Job - -try: - newDir = None - print os.uname() - beginTime = time.time() - print 'beginTime=%f' % beginTime - print "work in dir '@@tmpDir@@'" - sys.stdout.flush() - if not os.path.exists("@@tmpDir@@"): - raise IOError("ERROR: temporary directory '@@tmpDir@@' doesn't exist") - - fileSize = 0 - if not os.path.exists("@@groupId@@"): - @@cmdSize@@ - freeGigaNeededInTmpDir = float(1 + fileSize) - freeSpace = os.statvfs("@@tmpDir@@") - if ((freeSpace.f_bavail * freeSpace.f_frsize) / 1073741824.0 < freeGigaNeededInTmpDir): - raise RepetException("ERROR: less than %.2fG of free space in '@@tmpDir@@'" % freeGigaNeededInTmpDir) - - os.chdir("@@tmpDir@@") - if not os.path.exists("@@groupId@@"): - try: - os.mkdir("@@groupId@@") - except OSError, e : - if e.args[0] != 17: - raise RepetException("ERROR: can't create '@@groupId@@'") - os.chdir("@@groupId@@") - @@cmdCopy@@ - else: - os.chdir("@@groupId@@") - - newDir = "@@groupId@@_@@jobName@@_@@time@@" - if os.path.exists(newDir): - shutil.rmtree(newDir) - os.mkdir(newDir) - os.chdir(newDir) - - iJob = Job(jobname = "@@jobName@@", groupid = "@@groupId@@", launcherFile = "@@launcher@@", node = os.getenv("HOSTNAME")) - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "@@jobTableName@@") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "running") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - - @@cmdStart@@ - if log != 0: - raise RepetException("ERROR: job returned %i" % log) - else: - print "job finished successfully" - sys.stdout.flush() - @@cmdFinish@@ - - os.chdir("..") - shutil.rmtree(newDir) - - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "@@jobTableName@@") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "finished") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - - endTime = time.time() - print 'endTime=%f' % endTime - print 'executionTime=%f' % (endTime - beginTime) - print os.uname() - sys.stdout.flush() - -except IOError, e : - print e - iJob = Job(jobname = "@@jobName@@", groupid = "@@groupId@@", launcherFile = "@@launcher@@", node = os.getenv("HOSTNAME")) - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "@@jobTableName@@") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "error") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - sys.exit(1) - -except Exception, e : - print "tmpDir is : @@tmpDir@@" - print "cDir is : @@cDir@@" - print e - if newDir != None and os.path.exists("../%s" % newDir) and not os.path.exists("@@cDir@@/%s" % newDir): - os.chdir("..") - shutil.move(newDir, "@@cDir@@/%s" % newDir) - iJob = Job(jobname = "@@jobName@@", groupid = "@@groupId@@", launcherFile = "@@launcher@@", node = os.getenv("HOSTNAME")) - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "@@jobTableName@@") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "error") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - sys.exit(1)
--- a/commons/core/launcher/Launcher.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,229 +0,0 @@ -from commons.tools.CleanClusterNodesAfterRepet import CleanClusterNodesAfterRepet -from commons.core.stat.Stat import Stat -from commons.core.launcher.WriteScript import WriteScript -from commons.core.sql.TableJobAdaptatorFactory import TableJobAdaptatorFactory -from commons.core.sql.Job import Job -import stat -import os -import re -import sys -import time -import glob - -class Launcher(object): - - #TODO: remove unused parameters : query="", subject="", param="", job_table="" - def __init__( self, jobdb, query="", subject="", param="", cdir="", - tmpdir="", job_table="", queue="", groupid="", acro="X", - chooseTemplateWithCopy = False, chooseTemplateLight = False): - if jobdb.__class__.__name__ == "RepetJob": - self.jobdb = TableJobAdaptatorFactory.createInstance(jobdb, "jobs") - else: - self.jobdb = jobdb - self.jobdb.checkJobTable() - if cdir == "": - cdir = os.getcwd() - self.cdir = cdir - self.tmpdir = tmpdir - self.groupid = groupid - self.acronyme = acro - self._chooseTemplateWithCopy = chooseTemplateWithCopy - self._chooseTemplateLight = chooseTemplateLight - self.queue, self.lResources = self.getQueueNameAndResources(queue) - self._createJobInstance() - self._nbJobs = 0 - - def getQueueNameAndResources(self, configQueue): - tokens = configQueue.replace("'","").split(" ") - queueName = "" - lResources = [] - if tokens[0] != "": - if re.match(".*\.q", tokens[0]): - queueName = tokens[0] - lResources = tokens[1:] - else: - lResources = tokens - return queueName, lResources - - def createGroupidIfItNotExist(self): - if self.groupid == "": - self.job.groupid = str(os.getpid()) - else: - self.job.groupid = self.groupid - - def beginRun( self ): - self.createGroupidIfItNotExist() - if self.jobdb.hasUnfinishedJob(self.job.groupid): - self.jobdb.waitJobGroup(self.job.groupid) - else: - self.jobdb.cleanJobGroup(self.job.groupid) - - ## Launch one job in parallel - # - # @param cmdStart string command-line for the job to be launched - # @param cmdFinish string command to retrieve result files - # @warning the jobname has to be defined outside from this method - # - def runSingleJob(self, cmdStart, cmdFinish = "", cmdSize = "", cmdCopy = ""): - if self._nbJobs == 0: - self._nbJobs = 1 - pid = str(os.getpid()) - now = time.localtime() - #TODO: rename ClusterLauncher_ ... - pyFileName = self.cdir + "/ClusterLauncher_" + self.job.groupid + "_" +\ - self.job.jobname + "_" + str(now[0]) + "-" + str(now[1]) +\ - "-" + str(now[2]) + "_" + pid + ".py" - self.job.launcher = pyFileName - - #TODO: to remove when refactoring is done - cmdStart = self._indentCmd(cmdStart) - cmdFinish = self._indentCmd(cmdFinish) - - iWriteScript = WriteScript(self.job, self.jobdb, self.cdir, self.tmpdir, self._chooseTemplateWithCopy, self._chooseTemplateLight) - iWriteScript.run(cmdStart, cmdFinish, pyFileName, cmdSize, cmdCopy) - os.chmod(pyFileName, stat.S_IRWXU+stat.S_IRGRP+stat.S_IXGRP+stat.S_IROTH+stat.S_IXOTH) - sys.stdout.flush() - log = self.jobdb.submitJob(self.job) - if log != 0: - print "ERROR while submitting job to the cluster" - sys.exit(1) - - def endRun(self, cleanNodes = False): - string = "waiting for %i job(s) with groupid '%s' (%s)" % (self._nbJobs, self.job.groupid, time.strftime("%Y-%m-%d %H:%M:%S")) - print string; sys.stdout.flush() - self.jobdb.waitJobGroup(self.job.groupid) - if self._nbJobs > 1: - string = "all jobs with groupid '%s' are finished (%s)" % (self.job.groupid, time.strftime("%Y-%m-%d %H:%M:%S")) - print string; sys.stdout.flush() - - if cleanNodes: - string = "start cleaning cluster nodes (%s)" % time.strftime("%Y-%m-%d %H:%M:%S") - print string; sys.stdout.flush() - self.cleanNodes() - string = "end cleaning cluster nodes (%s)" % time.strftime("%Y-%m-%d %H:%M:%S") - print string; sys.stdout.flush() - - statsExecutionTime = self.getStatsOfExecutionTime() - if self._nbJobs > 1: - print "execution time of all jobs (seconds): %f" % statsExecutionTime.getSum() - print "execution time per job: %s" % statsExecutionTime.string() - sys.stdout.flush() - self.jobdb.cleanJobGroup(self.job.groupid) - - def getStatsOfExecutionTime(self, acronyme = ""): - stat = Stat() - if acronyme == "": - pattern = "%s*.o*" % self.acronyme - else: - pattern = "%s*.o*" % acronyme - lJobFiles = glob.glob(pattern) - for f in lJobFiles: - fH = open(f, "r") - while True: - line = fH.readline() - if line == "": - break - if "executionTime" in line: - stat.add( float(line[:-1].split("=")[1] ) ) - break - fH.close() - return stat - - def clean( self, acronyme = "", stdout = True, stderr = True ): - lFileToRemove = [] - if acronyme == "": - acronyme = self.acronyme - pattern = "ClusterLauncher*%s*.py" % ( acronyme ) - lFileToRemove.extend(glob.glob( pattern )) - if stdout: - pattern = "%s*.o*" % ( acronyme ) - lFileToRemove.extend(glob.glob( pattern )) - if stderr: - pattern = "%s*.e*" % ( acronyme ) - lFileToRemove.extend(glob.glob( pattern )) - for file in lFileToRemove: - os.remove(file) - - #TODO: handle of nodesMustBeCleaned => class attribute ? - def runLauncherForMultipleJobs(self, acronymPrefix, lCmdsTuples, cleanMustBeDone = True, nodesMustBeCleaned = False): - self.beginRun() - print "submitting job(s) with groupid '%s' (%s)" % (self.job.groupid, time.strftime("%Y-%m-%d %H:%M:%S")) - for cmdsTuple in lCmdsTuples: - self._nbJobs += 1 - self.acronyme = "%s_%s" % (acronymPrefix, self._nbJobs) - self.job.jobname = self.acronyme - if len(cmdsTuple) == 2: - self.runSingleJob(cmdsTuple[0], cmdsTuple[1]) - else: - self.runSingleJob(cmdsTuple[0], cmdsTuple[1], cmdsTuple[2], cmdsTuple[3]) - self._createJobInstance() - self.createGroupidIfItNotExist() - self.acronyme = acronymPrefix - self.endRun(nodesMustBeCleaned) - if cleanMustBeDone: - self.clean("%s_" % acronymPrefix) - self.jobdb.close() - - def prepareCommands(self, lCmds, lCmdStart = [], lCmdFinish = [], lCmdSize = [], lCmdCopy = []): - cmdStart = "" - for cmd in lCmdStart: - cmdStart += "%s\n\t" % cmd - for cmd in lCmds: - cmdStart += "%s\n\t" % cmd - cmdFinish = "" - for cmd in lCmdFinish: - cmdFinish += "%s\n\t" % cmd - cmdSize = "" - for cmd in lCmdSize: - cmdSize += "%s\n\t\t" % cmd - cmdCopy = "" - for cmd in lCmdCopy: - cmdCopy += "%s\n\t\t" % cmd - return (cmdStart, cmdFinish, cmdSize, cmdCopy) - - #TODO: to remove when refactoring is done - def prepareCommands_withoutIndentation(self, lCmds, lCmdStart = [], lCmdFinish = [], lCmdSize = [], lCmdCopy = []): - cmdStart = "" - for cmd in lCmdStart: - cmdStart += "%s\n" % cmd - for cmd in lCmds: - cmdStart += "%s\n" % cmd - cmdFinish = "" - for cmd in lCmdFinish: - cmdFinish += "%s\n" % cmd - cmdSize = "" - for cmd in lCmdSize: - cmdSize += "%s\n\t\t" % cmd - cmdCopy = "" - for cmd in lCmdCopy: - cmdCopy += "%s\n\t\t" % cmd - return (cmdStart, cmdFinish, cmdSize, cmdCopy) - - def getSystemCommand(self, prg, lArgs): - systemCmd = "log = os.system(\"" + prg - for arg in lArgs: - systemCmd += " " + arg - systemCmd += "\")" - return systemCmd - - def cleanNodes(self): - iCleanClusterNodeAfterRepet = CleanClusterNodesAfterRepet() - iCleanClusterNodeAfterRepet.setLNodes(self.jobdb.getNodesListByGroupId(self.groupid)) - iCleanClusterNodeAfterRepet.setTempDirectory(self.tmpdir) - iCleanClusterNodeAfterRepet.setPattern("%s*" % self.groupid) - iCleanClusterNodeAfterRepet.run() - - #TODO: to remove when refactoring is done - def _indentCmd(self, cmd): - lCmd = cmd.split("\n") - cmd_Tab = "%s\n" % lCmd[0] - for line in lCmd[1:-1]: - cmd_Tab += "\t%s\n" % line - return cmd_Tab - - def _createJobInstance(self): - if self.lResources == []: - #To have mem_free=1G: - self.job = Job(queue=self.queue) - else: - self.job = Job(queue=self.queue, lResources=self.lResources)
--- a/commons/core/launcher/Launcher2.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,294 +0,0 @@ -from commons.tools.CleanClusterNodesAfterRepet import CleanClusterNodesAfterRepet -from commons.core.stat.Stat import Stat -from commons.core.launcher.WriteScript import WriteScript -from commons.core.sql.TableJobAdaptatorFactory import TableJobAdaptatorFactory -from commons.core.sql.Job import Job -import stat -import os -import re -import sys -import time -import glob - -class LauncherParameter(object): - - def __init__(self, jobDB): - self._jobDB = jobDB - - def getJobDB(self): - return self._jobDB - - def setQuery(self, query): - self._query = query - - def setSubject(self, subject): - self._subject = subject - - def setParam(self, param): - self._param = param - - def setCurrentDir(self, currentDir): - self._currentDir = currentDir - - def getCurrentDir(self): - return self._currentDir - - def setTempDir(self, tempDir): - self._tempDir = tempDir - - def getTempDir(self): - return self._tempDir - - def setJobTable(self, jobTable): - self._jobTable = jobTable - - def setQueue(self, queue): - self._queue = queue - - def getQueue(self): - return self._queue - - def setGroupId(self, groupId): - self._groupId = groupId - - def getGroupId(self): - return self._groupId - - def setAcronym(self, acronym): - self._acronym = acronym - - def getAcronym(self): - return self._acronym - - @staticmethod - def createParameter(jobdb, groupid, acronym): - launcherParameter = LauncherParameter(jobdb) - launcherParameter.setQuery(os.getcwd()) - launcherParameter.setSubject("") - launcherParameter.setParam("") - launcherParameter.setCurrentDir(os.getcwd()) - launcherParameter.setTempDir(os.getcwd()) - launcherParameter.setJobTable("") - launcherParameter.setQueue("") - launcherParameter.setGroupId(groupid) - launcherParameter.setAcronym(acronym) - return launcherParameter - - -class Launcher2(object): - - #TODO: remove unused parameters : query="", subject="", param="", job_table="" - def __init__(self, iLauncherParameter): - jobdb = iLauncherParameter.getJobDB() - cdir = iLauncherParameter.getCurrentDir() - if jobdb.__class__.__name__ == "RepetJob": - self.jobdb = TableJobAdaptatorFactory.createInstance(jobdb, "jobs") - else: - self.jobdb = jobdb - self.jobdb.checkJobTable() - if cdir == "": - cdir = os.getcwd() - self.cdir = cdir - self.tmpdir = iLauncherParameter.getTempDir() - self.groupid = iLauncherParameter.getGroupId() - self.acronyme = iLauncherParameter.getAcronym() - self._chooseTemplateWithCopy = False - self._chooseTemplateLight = False - self.queue, self.lResources = self.getQueueNameAndResources(iLauncherParameter.getQueue()) - self._createJobInstance() - self._nbJobs = 0 - - def getQueueNameAndResources(self, configQueue): - tokens = configQueue.replace("'","").split(" ") - queueName = "" - lResources = [] - if tokens[0] != "": - if re.match(".*\.q", tokens[0]): - queueName = tokens[0] - lResources = tokens[1:] - else: - lResources = tokens - return queueName, lResources - - def createGroupidIfItNotExist(self): - if self.groupid == "": - self.job.groupid = str(os.getpid()) - else: - self.job.groupid = self.groupid - - def beginRun( self ): - self.createGroupidIfItNotExist() - if self.jobdb.hasUnfinishedJob(self.job.groupid): - self.jobdb.waitJobGroup(self.job.groupid) - else: - self.jobdb.cleanJobGroup(self.job.groupid) - - ## Launch one job in parallel - # - # @param cmdStart string command-line for the job to be launched - # @param cmdFinish string command to retrieve result files - # @warning the jobname has to be defined outside from this method - # - def runSingleJob(self, cmdStart, cmdFinish = "", cmdSize = "", cmdCopy = ""): - if self._nbJobs == 0: - self._nbJobs = 1 - pid = str(os.getpid()) - now = time.localtime() - #TODO: rename ClusterLauncher_ ... - pyFileName = self.cdir + "/ClusterLauncher_" + self.job.groupid + "_" +\ - self.job.jobname + "_" + str(now[0]) + "-" + str(now[1]) +\ - "-" + str(now[2]) + "_" + pid + ".py" - self.job.launcher = pyFileName - - #TODO: to remove when refactoring is done - cmdStart = self._indentCmd(cmdStart) - cmdFinish = self._indentCmd(cmdFinish) - - iWriteScript = WriteScript(self.job, self.jobdb, self.cdir, self.tmpdir, self._chooseTemplateWithCopy, self._chooseTemplateLight) - iWriteScript.run(cmdStart, cmdFinish, pyFileName, cmdSize, cmdCopy) - os.chmod(pyFileName, stat.S_IRWXU+stat.S_IRGRP+stat.S_IXGRP+stat.S_IROTH+stat.S_IXOTH) - sys.stdout.flush() - log = self.jobdb.submitJob(self.job) - if log != 0: - print "ERROR while submitting job to the cluster" - sys.exit(1) - - def endRun(self, cleanNodes = False): - string = "waiting for %i job(s) with groupid '%s' (%s)" % (self._nbJobs, self.job.groupid, time.strftime("%Y-%m-%d %H:%M:%S")) - print string; sys.stdout.flush() - self.jobdb.waitJobGroup(self.job.groupid) - if self._nbJobs > 1: - string = "all jobs with groupid '%s' are finished (%s)" % (self.job.groupid, time.strftime("%Y-%m-%d %H:%M:%S")) - print string; sys.stdout.flush() - - if cleanNodes: - string = "start cleaning cluster nodes (%s)" % time.strftime("%Y-%m-%d %H:%M:%S") - print string; sys.stdout.flush() - self.cleanNodes() - string = "end cleaning cluster nodes (%s)" % time.strftime("%Y-%m-%d %H:%M:%S") - print string; sys.stdout.flush() - - statsExecutionTime = self.getStatsOfExecutionTime() - if self._nbJobs > 1: - print "execution time of all jobs (seconds): %f" % statsExecutionTime.getSum() - print "execution time per job: %s" % statsExecutionTime.string() - sys.stdout.flush() - self.jobdb.cleanJobGroup(self.job.groupid) - - def getStatsOfExecutionTime(self, acronyme = ""): - stat = Stat() - if acronyme == "": - pattern = "%s*.o*" % self.acronyme - else: - pattern = "%s*.o*" % acronyme - lJobFiles = glob.glob(pattern) - for f in lJobFiles: - fH = open(f, "r") - while True: - line = fH.readline() - if line == "": - break - if "executionTime" in line: - stat.add( float(line[:-1].split("=")[1] ) ) - break - fH.close() - return stat - - def clean( self, acronyme = "", stdout = True, stderr = True ): - lFileToRemove = [] - if acronyme == "": - acronyme = self.acronyme - pattern = "ClusterLauncher*%s*.py" % ( acronyme ) - lFileToRemove.extend(glob.glob( pattern )) - if stdout: - pattern = "%s*.o*" % ( acronyme ) - lFileToRemove.extend(glob.glob( pattern )) - if stderr: - pattern = "%s*.e*" % ( acronyme ) - lFileToRemove.extend(glob.glob( pattern )) - for file in lFileToRemove: - os.remove(file) - - #TODO: handle of nodesMustBeCleaned => class attribute ? - def runLauncherForMultipleJobs(self, acronymPrefix, lCmdsTuples, cleanMustBeDone = True, nodesMustBeCleaned = False): - self.beginRun() - print "submitting job(s) with groupid '%s' (%s)" % (self.job.groupid, time.strftime("%Y-%m-%d %H:%M:%S")) - for cmdsTuple in lCmdsTuples: - self._nbJobs += 1 - self.acronyme = "%s_%s" % (acronymPrefix, self._nbJobs) - self.job.jobname = self.acronyme - if len(cmdsTuple) == 2: - self.runSingleJob(cmdsTuple[0], cmdsTuple[1]) - else: - self.runSingleJob(cmdsTuple[0], cmdsTuple[1], cmdsTuple[2], cmdsTuple[3]) - self._createJobInstance() - self.createGroupidIfItNotExist() - self.acronyme = acronymPrefix - self.endRun(nodesMustBeCleaned) - if cleanMustBeDone: - self.clean("%s_" % acronymPrefix) - self.jobdb.close() - - def prepareCommands(self, lCmds, lCmdStart = [], lCmdFinish = [], lCmdSize = [], lCmdCopy = []): - cmdStart = "" - for cmd in lCmdStart: - cmdStart += "%s\n\t" % cmd - for cmd in lCmds: - cmdStart += "%s\n\t" % cmd - cmdFinish = "" - for cmd in lCmdFinish: - cmdFinish += "%s\n\t" % cmd - cmdSize = "" - for cmd in lCmdSize: - cmdSize += "%s\n\t\t" % cmd - cmdCopy = "" - for cmd in lCmdCopy: - cmdCopy += "%s\n\t\t" % cmd - return (cmdStart, cmdFinish, cmdSize, cmdCopy) - - #TODO: to remove when refactoring is done - def prepareCommands_withoutIndentation(self, lCmds, lCmdStart = [], lCmdFinish = [], lCmdSize = [], lCmdCopy = []): - cmdStart = "" - for cmd in lCmdStart: - cmdStart += "%s\n" % cmd - for cmd in lCmds: - cmdStart += "%s\n" % cmd - cmdFinish = "" - for cmd in lCmdFinish: - cmdFinish += "%s\n" % cmd - cmdSize = "" - for cmd in lCmdSize: - cmdSize += "%s\n\t\t" % cmd - cmdCopy = "" - for cmd in lCmdCopy: - cmdCopy += "%s\n\t\t" % cmd - return (cmdStart, cmdFinish, cmdSize, cmdCopy) - - def getSystemCommand(self, prg, lArgs): - systemCmd = "log = os.system(\"" + prg - for arg in lArgs: - systemCmd += " " + arg - systemCmd += "\")" - return systemCmd - - def cleanNodes(self): - iCleanClusterNodeAfterRepet = CleanClusterNodesAfterRepet() - iCleanClusterNodeAfterRepet.setLNodes(self.jobdb.getNodesListByGroupId(self.groupid)) - iCleanClusterNodeAfterRepet.setTempDirectory(self.tmpdir) - iCleanClusterNodeAfterRepet.setPattern("%s*" % self.groupid) - iCleanClusterNodeAfterRepet.run() - - #TODO: to remove when refactoring is done - def _indentCmd(self, cmd): - lCmd = cmd.split("\n") - cmd_Tab = "%s\n" % lCmd[0] - for line in lCmd[1:-1]: - cmd_Tab += "\t%s\n" % line - return cmd_Tab - - def _createJobInstance(self): - if self.lResources == []: - #To have mem_free=1G: - self.job = Job(queue=self.queue) - else: - self.job = Job(queue=self.queue, lResources=self.lResources)
--- a/commons/core/launcher/LauncherUtils.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,31 +0,0 @@ -class LauncherUtils(object): - - @staticmethod - def createHomogeneousSizeList(lStringSizeTuples, maxSize): - lStringSizeTuplesSorted = sorted(lStringSizeTuples, key=lambda stringSizeTuple:(stringSizeTuple[1], stringSizeTuple[0]), reverse = True) - lStringSizeList = [] - lStringSize = [] - sumTupleSize = 0 - iteratorFromBegin = 0 - iteratorFromEnd = len(lStringSizeTuplesSorted) - 1 - for tuple in lStringSizeTuplesSorted: - if sumTupleSize + tuple[1] < maxSize: - lStringSize.append(tuple[0]) - sumTupleSize += tuple[1] - elif tuple[1] >= maxSize: - lStringSizeList.append([tuple[0]]) - else: - tupleFromEnd = lStringSizeTuplesSorted[iteratorFromEnd] - while sumTupleSize + tupleFromEnd[1] < maxSize and iteratorFromBegin < iteratorFromEnd: - lStringSize.append(tupleFromEnd[0]) - sumTupleSize += tupleFromEnd[1] - del lStringSizeTuplesSorted[iteratorFromEnd] - iteratorFromEnd -= 1 - tupleFromEnd = lStringSizeTuplesSorted[iteratorFromEnd] - lStringSizeList.append(lStringSize) - lStringSize = [tuple[0]] - sumTupleSize = tuple[1] - iteratorFromBegin += 1 - if lStringSize: - lStringSizeList.append(lStringSize) - return lStringSizeList \ No newline at end of file
--- a/commons/core/launcher/WriteScript.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,76 +0,0 @@ -import os -import time - -class WriteScript(object): - - def __init__(self, job = None, jobdb = None, cdir = "", tmpdir = "", chooseTemplateWithCopy = False, chooseTemplateLight = False): - self._iJob = job - self._iJobdb = jobdb - self._cDir = cdir - self._tmpDir = tmpdir - self._chooseTemplateWithCopy = chooseTemplateWithCopy - self._chooseTemplateLight = chooseTemplateLight - - def run(self, cmdStart, cmdFinish, pyFileName, cmdSize = "", cmdCopy = ""): - if self._chooseTemplateLight: - d = self.createJobScriptLightDict(cmdStart, cmdFinish, cmdSize, cmdCopy) - else: - d = self.createJobScriptDict(cmdStart, cmdFinish, cmdSize, cmdCopy) - self.fillTemplate(pyFileName, d) - - def fillTemplate(self, outputFileName, dict): - if self._chooseTemplateWithCopy: - inputFileName = "%s/commons/core/launcher/JobScriptWithFilesCopyTemplate.py" % os.environ["REPET_PATH"] - else: - inputFileName = "%s/commons/core/launcher/JobScriptTemplate.py" % os.environ["REPET_PATH"] - - if self._chooseTemplateLight: - inputFileName = "%s/commons/core/launcher/JobScriptTemplateLight.py" % os.environ["REPET_PATH"] - - input = open(inputFileName, "r") - data = input.read() - input.close() - for key, value in dict.items(): - data = data.replace("@@%s@@" % key, value) - output = open(outputFileName, "w") - output.write(data) - output.close() - - def createJobScriptDict(self, cmdStart, cmdFinish, cmdSize, cmdCopy): - dict = { - "tmpDir" : self._tmpDir, - "jobTableName" : self._iJobdb._table, - "groupId" : self._iJob.groupid, - "jobName" : self._iJob.jobname, - "launcher" : self._iJob.launcher, - "time" : time.strftime("%Y%m%d-%H%M%S"), - "repet_path" : os.environ["REPET_PATH"], - "repet_host" : os.environ["REPET_HOST"], - "repet_user" : os.environ["REPET_USER"], - "repet_pw" : os.environ["REPET_PW"], - "repet_db" : os.environ["REPET_DB"], - "repet_port" : os.environ["REPET_PORT"], - "cmdStart" : cmdStart, - "cmdFinish" : cmdFinish, - "cDir" : self._cDir, - "cmdSize" : cmdSize, - "cmdCopy" : cmdCopy - } - return dict - - def createJobScriptLightDict(self, cmdStart, cmdFinish, cmdSize, cmdCopy): - dict = { - "tmpDir" : self._tmpDir, - "jobTableName" : self._iJobdb._table, - "groupId" : self._iJob.groupid, - "jobName" : self._iJob.jobname, - "launcher" : self._iJob.launcher, - "time" : time.strftime("%Y%m%d-%H%M%S"), - "repet_path" : os.environ["REPET_PATH"], - "cmdStart" : cmdStart, - "cmdFinish" : cmdFinish, - "cDir" : self._cDir, - "cmdSize" : cmdSize, - "cmdCopy" : cmdCopy - } - return dict
--- a/commons/core/launcher/test/Test_Launcher.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,330 +0,0 @@ -from commons.core.utils.FileUtils import FileUtils -from commons.core.launcher.Launcher import Launcher -from commons.core.launcher.WriteScript import WriteScript -from commons.core.stat.Stat import Stat -from commons.core.sql.TableJobAdaptatorFactory import TableJobAdaptatorFactory -from commons.core.sql.DbFactory import DbFactory -from commons.core.sql.Job import Job -import unittest -import os -import shutil -import time -import stat - -#TODO: Test_F_Launcher.py : to execute prepareCommands() and runSingleJob() -# to test runLauncherForMultipleJobs() -#TODO: check clean of "Test_runSingleJob" -#TODO: refactoring => choose between "self._queue" or "lResources" to set resources -class Test_Launcher(unittest.TestCase): - - SARUMAN_NAME = "compute-2-46.local" - - def setUp(self): - self._cDir = os.getcwd() - self._tmpDir = self._cDir - self._groupid = "test" - self._jobTable = "dummyJobTable" - self._iDb = DbFactory.createInstance() - self._iDb.createTable(self._jobTable, "jobs", overwrite = True) - self._jobdb = TableJobAdaptatorFactory.createInstance(self._iDb, self._jobTable) - self._queue = "" - self._configFileName = "dummyConfigFile" - - def tearDown(self): - self._iDb.dropTable(self._jobTable) - self._iDb.close() - FileUtils.removeFilesByPattern('*.e*') - FileUtils.removeFilesByPattern('*.o*') - FileUtils.removeFilesByPattern('launcherFileTest_BeginRun.py') - FileUtils.removeFilesByPattern(self._configFileName) - FileUtils.removeFilesByPattern('ClusterLauncher_*') - - def test__init__wrong_fields_for_job_table(self): - self._iDb.dropTable(self._jobTable) - sqlCmd = "CREATE TABLE " + self._jobTable - sqlCmd += " ( jobid INT UNSIGNED" - sqlCmd += ", jobname VARCHAR(255)" - sqlCmd += ", groupid VARCHAR(255)" - sqlCmd += ", command TEXT" - sqlCmd += ", launcher VARCHAR(1024)" - sqlCmd += ", queue VARCHAR(255)" - sqlCmd += ", status VARCHAR(255)" - sqlCmd += ", time DATETIME" - sqlCmd += ", node VARCHAR(255) )" - self._iDb.execute(sqlCmd) - acronym = "Test__init__" - iLauncher = Launcher(self._jobdb, os.getcwd(), "", "", self._cDir, self._tmpDir, "", self._queue, self._groupid, acronym) - lExpFields = sorted(["jobid", "jobname", "groupid", "launcher", "queue", "resources", "status", "time", "node"]) - lObsFields = sorted(self._iDb.getFieldList(self._jobTable)) - self.assertEquals(lExpFields, lObsFields) - expJob = Job(queue = self._queue) - obsJob = iLauncher.job - self.assertEquals(expJob, obsJob) - - def test__init__withResources(self): - queue = "main.q mem_free=3G" - acronym = "Test__init__" - expQueue = "main.q" - explResources = ['mem_free=3G'] - expJob = Job(queue = expQueue, lResources = explResources) - iLauncher = Launcher(self._jobdb, os.getcwd(), "", "", self._cDir, self._tmpDir, "", queue, self._groupid, acronym) - obsJob = iLauncher.job - self.assertEquals(expJob, obsJob) - - def test_createGroupidIfItNotExist(self): - acronym = "checkGroupID" - iLauncher = Launcher(self._jobdb, os.getcwd(), "", "", self._cDir, self._tmpDir, "", self._queue, self._groupid, acronym) - iLauncher.createGroupidIfItNotExist() - obsGroupid = iLauncher.job.groupid - self.assertEquals(self._groupid, obsGroupid) - - def test_createGroupidIfItNotExist_without_groupid(self): - groupid = "" - acronym = "checkGroupID" - iLauncher = Launcher(self._jobdb, os.getcwd(), "", "", self._cDir, self._tmpDir, "", self._queue, groupid, acronym) - iLauncher.createGroupidIfItNotExist() - obsGroupid = iLauncher.job.groupid - self.assertTrue(obsGroupid != "") - - def test_beginRun_with_Job_finished_in_Table(self): - acronym = "BeginRun" - iJob = Job(queue = self._queue) - self._jobdb.recordJob(iJob) - self._jobdb.changeJobStatus(iJob, "finished") - iLauncher = Launcher(self._jobdb, os.getcwd(), "", "", self._cDir, self._tmpDir, "", self._queue, self._groupid, acronym) - iLauncher.beginRun() - self.assertTrue(self._jobdb.getCountStatus(self._groupid, "finished") == 0) - - def test_beginRun_with_Job_unfinished_in_Table(self): - acronym = "testU_BeginRun" - cmd_start = "log = os.system( \"date;sleep 10;date\" )\n" - pyFileName = "%s/launcherFileTest_BeginRun.py" % os.getcwd() - if Test_Launcher.SARUMAN_NAME == os.getenv("HOSTNAME"): - iJob = Job(1, acronym, self._groupid, "", cmd_start, pyFileName, lResources=["test=TRUE"]) - else: - iJob = Job(1, acronym, self._groupid, "", cmd_start, pyFileName) - iWriteScript = WriteScript(iJob, self._jobdb, self._cDir, self._tmpDir) - iWriteScript.run(cmd_start, "", pyFileName) - os.chmod(pyFileName, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) - self._jobdb.submitJob(iJob) - iLauncher = Launcher(self._jobdb, os.getcwd(), "", "", self._cDir, self._tmpDir, "", self._queue, self._groupid, acronym) - - iLauncher.beginRun() - - self.assertTrue(self._jobdb.getCountStatus(self._groupid, "finished") == 1) - - def test_getStatsOfExecutionTime(self): - acronym = "test_statTime" - - expLValues = [1000.00000, 1000.00000] - expStat = Stat(expLValues) - - f = open(acronym +".o1", "w") - f.write("executionTime=1000.000000") - f.close() - f = open(acronym +".o2", "w") - f.write("executionTime=1000.000000") - f.close() - - iLauncher = Launcher(self._jobdb, os.getcwd(), "", "", self._cDir, self._tmpDir, "", self._queue, self._groupid, acronym) - obsStat = iLauncher.getStatsOfExecutionTime(acronym) - - self.assertEqual(expStat, obsStat) - - def test_endRun(self): - acronym = "testU_EndRun" - cmd_start = "log = os.system( \"date;sleep 10;date\" )\n" - pyFileName = "%s/launcherFileTest_EndRun.py" % os.getcwd() - if Test_Launcher.SARUMAN_NAME == os.getenv("HOSTNAME"): - iJob = Job(1, acronym, self._groupid, "", cmd_start, pyFileName, lResources=["test=TRUE"]) - else: - iJob = Job(1, acronym, self._groupid, "", cmd_start, pyFileName) - - iWriteScript = WriteScript(iJob, self._jobdb, self._cDir, self._tmpDir) - iWriteScript.run(cmd_start, "", pyFileName) - os.chmod(pyFileName, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) - self._jobdb.submitJob(iJob) - iLauncher = Launcher(self._jobdb, os.getcwd(), "", "", self._cDir, self._tmpDir, "", self._queue, self._groupid, acronym) - iLauncher.job.groupid = self._groupid - iLauncher.endRun() - - self.assertTrue(self._jobdb.getCountStatus(self._groupid, "finished") == 0) - self.assertTrue(self._jobdb.getCountStatus(self._groupid, "error") == 0) - self.assertTrue(self._jobdb.getCountStatus(self._groupid, "waiting") == 0) - - os.remove(iJob.launcher) - - def test_clean(self): - acronym = "test_clean" - f = open("ClusterLauncher" + acronym + ".py", "w") - f.close() - f = open(acronym + ".o1", "w") - f.close() - f = open(acronym + ".e1", "w") - f.close() - iLauncher = Launcher(self._jobdb, os.getcwd(), "", "", self._cDir, self._tmpDir, "", self._queue, self._groupid, acronym) - iLauncher.clean(acronym) - self.assertFalse(FileUtils.isRessourceExists("ClusterLauncher" + acronym + ".py")) - - def test_clean_without_acronym(self): - acronym = "" - acronym2 = "toto" - f = open("ClusterLauncher" + acronym2 + ".py", "w") - f.close() - f = open(acronym2 + ".o1", "w") - f.close() - f = open(acronym2 + ".e1", "w") - f.close() - iLauncher = Launcher(self._jobdb, os.getcwd(), "", "", self._cDir, self._tmpDir, "", self._queue, self._groupid, acronym2) - iLauncher.clean(acronym) - self.assertFalse(FileUtils.isRessourceExists("ClusterLauncher" + acronym2 + ".py")) - - def test_getQueueNameAndResources_queue_no_resource(self): - configQueue = "all.q" - expQueueName = "all.q" - expResources = [] - iLauncher = Launcher(self._jobdb) - obsQueueName, obsResources = iLauncher.getQueueNameAndResources(configQueue) - self.assertEquals(expQueueName, obsQueueName) - self.assertEquals(expResources, obsResources) - - def test_getQueueNameAndResources_queue_one_resource(self): - configQueue = "test.q 'test=TRUE'" - expQueueName = "test.q" - expResources = ["test=TRUE"] - iLauncher = Launcher(self._jobdb) - obsQueueName, obsResources = iLauncher.getQueueNameAndResources(configQueue) - self.assertEquals(expQueueName, obsQueueName) - self.assertEquals(expResources, obsResources) - - def test_getQueueNameAndResources_queue_two_resources(self): - configQueue = "big.q 's_data=8G s_cpu=96:00:00'" - expQueueName = "big.q" - expResources = ["s_data=8G", "s_cpu=96:00:00"] - iLauncher = Launcher(self._jobdb) - obsQueueName, obsResources = iLauncher.getQueueNameAndResources(configQueue) - self.assertEquals(expQueueName, obsQueueName) - self.assertEquals(expResources, obsResources) - - def test_getQueueNameAndResources_no_queue_no_resource(self): - configQueue = "" - expQueueName = "" - expResources = [] - iLauncher = Launcher(self._jobdb) - obsQueueName, obsResources = iLauncher.getQueueNameAndResources(configQueue) - self.assertEquals(expQueueName, obsQueueName) - self.assertEquals(expResources, obsResources) - - def test_getQueueNameAndResources_no_queue_one_resource(self): - configQueue = "s_data=8G" - expQueueName = "" - expResources = ["s_data=8G"] - iLauncher = Launcher(self._jobdb) - obsQueueName, obsResources = iLauncher.getQueueNameAndResources(configQueue) - self.assertEquals(expQueueName, obsQueueName) - self.assertEquals(expResources, obsResources) - - def test_getQueueNameAndResources_no_queue_two_resource(self): - configQueue = "s_data=8G s_cpu=96:00:00" - expQueueName = "" - expResources = ["s_data=8G", "s_cpu=96:00:00"] - iLauncher = Launcher(self._jobdb) - obsQueueName, obsResources = iLauncher.getQueueNameAndResources(configQueue) - self.assertEquals(expQueueName, obsQueueName) - self.assertEquals(expResources, obsResources) - -# #TODO: test with at least 2 lines in cmd - def test_runSingleJob(self): - acronym = "Test_runSingleJob" - os.mkdir(acronym) - os.chdir(acronym) - iLauncher = Launcher(self._jobdb, os.getcwd(), "", "", os.getcwd(), self._tmpDir, "", self._queue, self._groupid, acronym) - iLauncher.job.groupid = self._groupid - iLauncher.job.jobname = acronym - iLauncher.job.queue = self._queue - if Test_Launcher.SARUMAN_NAME == os.getenv("HOSTNAME"): - iLauncher.job.lResources = ["test=TRUE"] - cmd = "log = os.system(\"touch 'YuFei'\")\n" - iLauncher.runSingleJob(cmd) - time.sleep(20) - jobStatus = self._jobdb.getJobStatus(iLauncher.job) - os.chdir(self._cDir) - shutil.rmtree(acronym) - self.assertEqual(jobStatus, "finished") - - def test_runSingleJob_catch_error_wrong_tmpDir(self): - acronym = "Test_runSingleJob_catch_error" - os.mkdir(acronym) - os.chdir(acronym) - iLauncher = Launcher(self._jobdb, os.getcwd(), "", "", os.getcwd(), "%s/toto" % self._tmpDir, "", self._queue, self._groupid, acronym) - iLauncher.job.groupid = self._groupid - iLauncher.job.jobname = acronym - iLauncher.job.queue = self._queue - if Test_Launcher.SARUMAN_NAME == os.getenv("HOSTNAME"): - iLauncher.job.lResources = ["test=TRUE"] - cmd = "log = os.system(\"touch 'YuFei'\")\n" - iLauncher.runSingleJob(cmd) - time.sleep(20) - jobStatus = self._jobdb.getJobStatus(iLauncher.job) - os.chdir(self._cDir) - shutil.rmtree(acronym) - self.assertEqual(jobStatus, "error") - - def test_runSingleJob_catch_error_wrong_cmd(self): - acronym = "Test_runSingleJob_catch_error" - os.mkdir(acronym) - os.chdir(acronym) - iLauncher = Launcher(self._jobdb, os.getcwd(), "", "", os.getcwd(), self._tmpDir, "", self._queue, self._groupid, acronym) - iLauncher.job.groupid = self._groupid - iLauncher.job.jobname = acronym - iLauncher.job.queue = self._queue - if Test_Launcher.SARUMAN_NAME == os.getenv("HOSTNAME"): - iLauncher.job.lResources = ["test=TRUE"] - cmd = "log = os.system(\"truc -i toto\")\n" - iLauncher.runSingleJob(cmd) - time.sleep(20) - jobStatus = self._jobdb.getJobStatus(iLauncher.job) - self._jobdb.cleanJobGroup(self._groupid) - os.chdir(self._cDir) - shutil.rmtree(acronym) - self.assertEqual(jobStatus, "error") - - def test_prepareCommands(self): - expCmdStart = "os.symlink(\"../Yufei_chunks.fa\", \"Yufei_chunks.fa\")\n\tos.symlink(\"../Yufei_chunks.fa_cut\", \"Yufei_chunks.fa_cut\")\n\tlog = os.system(\"touch file\")\n\t" - expCmdFinish = "if os.path.exists(\"yufei.align\"):\n\t\tshutil.move(\"yufei.align\", \"yufeiLuo/.\" )\n\t" - expCmdSize = "fileSize = 3.2\n\t\t" - expCmdCopy = "shutil.copy(\"PY/Yufei_db/Yufei_chunks.fa\", \".\")\n\t\tshutil.copy(\"PY/Yufei_db/Yufei_chunks.fa_cut\", \".\")\n\t\t" - - lCmdStart = [] - lCmdStart.append("os.symlink(\"../Yufei_chunks.fa\", \"Yufei_chunks.fa\")") - lCmdStart.append("os.symlink(\"../Yufei_chunks.fa_cut\", \"Yufei_chunks.fa_cut\")") - lCmds = [] - lCmds.append("log = os.system(\"touch file\")") - lCmdFinish = [] - lCmdFinish.append("if os.path.exists(\"yufei.align\"):") - lCmdFinish.append("\tshutil.move(\"yufei.align\", \"yufeiLuo/.\" )") - lCmdSize = [] - lCmdSize.append("fileSize = 3.2") - lCmdCopy = [] - lCmdCopy.append("shutil.copy(\"PY/Yufei_db/Yufei_chunks.fa\", \".\")") - lCmdCopy.append("shutil.copy(\"PY/Yufei_db/Yufei_chunks.fa_cut\", \".\")") - - iLauncher = Launcher(self._jobdb) - obsCmdStart, obsCmdFinish, obsCmdSize, obsCmdCopy = iLauncher.prepareCommands(lCmds, lCmdStart, lCmdFinish, lCmdSize, lCmdCopy) - - self.assertEquals(expCmdStart, obsCmdStart) - self.assertEquals(expCmdFinish, obsCmdFinish) - self.assertEquals(expCmdSize, obsCmdSize) - self.assertEquals(expCmdCopy, obsCmdCopy) - - def test_getSystemCommand(self): - prg = "touch" - lArgs = [] - lArgs.append("file") - expCmd = "log = os.system(\"touch file\")" - iLauncher = Launcher(self._jobdb) - obsCmd = iLauncher.getSystemCommand(prg, lArgs) - self.assertEquals(expCmd, obsCmd) - -if __name__ == "__main__": - unittest.main() \ No newline at end of file
--- a/commons/core/launcher/test/Test_Launcher2.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,356 +0,0 @@ -from commons.core.utils.FileUtils import FileUtils -from commons.core.launcher.Launcher2 import Launcher2 -from commons.core.launcher.Launcher2 import LauncherParameter -from commons.core.launcher.WriteScript import WriteScript -from commons.core.stat.Stat import Stat -from commons.core.sql.TableJobAdaptatorFactory import TableJobAdaptatorFactory -from commons.core.sql.DbFactory import DbFactory -from commons.core.sql.Job import Job -import unittest -import os -import shutil -import time -import stat - -#TODO: Test_F_Launcher2.py : to execute prepareCommands() and runSingleJob() -# to test runLauncher2ForMultipleJobs() -#TODO: check clean of "Test_runSingleJob" -#TODO: refactoring => choose between "self._queue" or "lResources" to set resources -class Test_Launcher2(unittest.TestCase): - - SARUMAN_NAME = "compute-2-46.local" - - def setUp(self): - self._cDir = os.getcwd() - self._tmpDir = self._cDir - self._groupid = "test" - self._jobTable = "dummyJobTable" - self._iDb = DbFactory.createInstance() - self._iDb.createTable(self._jobTable, "jobs", overwrite = True) - self._jobdb = TableJobAdaptatorFactory.createInstance(self._iDb, self._jobTable) - self._queue = "" - self._configFileName = "dummyConfigFile" - - def tearDown(self): - self._iDb.dropTable(self._jobTable) - self._iDb.close() - FileUtils.removeFilesByPattern('*.e*') - FileUtils.removeFilesByPattern('*.o*') - FileUtils.removeFilesByPattern('Launcher2FileTest_BeginRun.py') - FileUtils.removeFilesByPattern(self._configFileName) - FileUtils.removeFilesByPattern('ClusterLauncher2_*') - - def test__init__wrong_fields_for_job_table(self): - self._iDb.dropTable(self._jobTable) - sqlCmd = "CREATE TABLE " + self._jobTable - sqlCmd += " ( jobid INT UNSIGNED" - sqlCmd += ", jobname VARCHAR(255)" - sqlCmd += ", groupid VARCHAR(255)" - sqlCmd += ", command TEXT" - sqlCmd += ", Launcher2 VARCHAR(1024)" - sqlCmd += ", queue VARCHAR(255)" - sqlCmd += ", status VARCHAR(255)" - sqlCmd += ", time DATETIME" - sqlCmd += ", node VARCHAR(255) )" - self._iDb.execute(sqlCmd) - acronym = "Test__init__" - launcherParameter = LauncherParameter.createParameter(self._jobdb, self._groupid, acronym) - iLauncher2 = Launcher2(launcherParameter) - - - lExpFields = sorted(["jobid", "jobname", "groupid", "launcher", "queue", "resources", "status", "time", "node"]) - lObsFields = sorted(self._iDb.getFieldList(self._jobTable)) - self.assertEquals(lExpFields, lObsFields) - expJob = Job(queue = self._queue) - obsJob = iLauncher2.job - self.assertEquals(expJob, obsJob) - - def test__init__withResources(self): - queue = "main.q mem_free=3G" - acronym = "Test__init__" - expQueue = "main.q" - explResources = ['mem_free=3G'] - expJob = Job(queue = expQueue, lResources = explResources) - - launcherParameter = LauncherParameter.createParameter(self._jobdb, self._groupid, acronym); - launcherParameter.setQueue(queue) - iLauncher2 = Launcher2(launcherParameter) - - obsJob = iLauncher2.job - self.assertEquals(expJob, obsJob) - - def test_createGroupidIfItNotExist(self): - acronym = "checkGroupID" - - launcherParameter = LauncherParameter.createParameter(self._jobdb, self._groupid, acronym); - iLauncher2 = Launcher2(launcherParameter) - iLauncher2.createGroupidIfItNotExist() - obsGroupid = iLauncher2.job.groupid - self.assertEquals(self._groupid, obsGroupid) - - def test_createGroupidIfItNotExist_without_groupid(self): - groupid = "" - acronym = "checkGroupID" - launcherParameter = LauncherParameter.createParameter(self._jobdb, self._groupid, acronym); - iLauncher2 = Launcher2(launcherParameter) - iLauncher2.createGroupidIfItNotExist() - obsGroupid = iLauncher2.job.groupid - self.assertTrue(obsGroupid != "") - - def test_beginRun_with_Job_finished_in_Table(self): - acronym = "BeginRun" - iJob = Job(queue = self._queue) - self._jobdb.recordJob(iJob) - self._jobdb.changeJobStatus(iJob, "finished") - - launcherParameter = LauncherParameter.createParameter(self._jobdb, self._groupid, acronym); - iLauncher2 = Launcher2(launcherParameter) - - iLauncher2.beginRun() - self.assertTrue(self._jobdb.getCountStatus(self._groupid, "finished") == 0) - -# def test_beginRun_with_Job_unfinished_in_Table(self): -# acronym = "testU_BeginRun" -# cmd_start = "log = os.system( \"date;sleep 10;date\" )\n" -# pyFileName = "%s/Launcher22FileTest_BeginRun.py" % os.getcwd() -# if Test_Launcher2.SARUMAN_NAME == os.getenv("HOSTNAME"): -# iJob = Job(1, acronym, self._groupid, "", cmd_start, pyFileName, lResources=["test=TRUE"]) -# else: -# iJob = Job(1, acronym, self._groupid, "", cmd_start, pyFileName) -# iWriteScript = WriteScript(iJob, self._jobdb, self._cDir, self._tmpDir) -# iWriteScript.run(cmd_start, "", pyFileName) -# os.chmod(pyFileName, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) -# self._jobdb.submitJob(iJob) - -# launcherParameter = LauncherParameter.createParameter(self._jobdb, self._groupid, acronym); -# iLauncher2 = Launcher2(launcherParameter) - -# iLauncher2.beginRun() - -# self.assertTrue(self._jobdb.getCountStatus(self._groupid, "finished") == 1) -# - def test_getStatsOfExecutionTime(self): - acronym = "test_statTime" - - expLValues = [1000.00000, 1000.00000] - expStat = Stat(expLValues) - - f = open(acronym +".o1", "w") - f.write("executionTime=1000.000000") - f.close() - f = open(acronym +".o2", "w") - f.write("executionTime=1000.000000") - f.close() - - launcherParameter = LauncherParameter.createParameter(self._jobdb, self._groupid, acronym); - iLauncher2 = Launcher2(launcherParameter) - obsStat = iLauncher2.getStatsOfExecutionTime(acronym) - - self.assertEqual(expStat, obsStat) -# -# def test_endRun(self): -# acronym = "testU_EndRun" -# cmd_start = "log = os.system( \"date;sleep 10;date\" )\n" -# pyFileName = "%s/Launcher22FileTest_EndRun.py" % os.getcwd() -# if Test_Launcher2.SARUMAN_NAME == os.getenv("HOSTNAME"): -# iJob = Job(1, acronym, self._groupid, "", cmd_start, pyFileName, lResources=["test=TRUE"]) -# else: -# iJob = Job(1, acronym, self._groupid, "", cmd_start, pyFileName) -# -# iWriteScript = WriteScript(iJob, self._jobdb, self._cDir, self._tmpDir) -# iWriteScript.run(cmd_start, "", pyFileName) -# os.chmod(pyFileName, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) -# self._jobdb.submitJob(iJob) -# iLauncher2 = Launcher2(self._jobdb, os.getcwd(), "", "", self._cDir, self._tmpDir, "", self._queue, self._groupid, acronym) -# iLauncher2.job.groupid = self._groupid -# iLauncher2.endRun() -# -# self.assertTrue(self._jobdb.getCountStatus(self._groupid, "finished") == 0) -# self.assertTrue(self._jobdb.getCountStatus(self._groupid, "error") == 0) -# self.assertTrue(self._jobdb.getCountStatus(self._groupid, "waiting") == 0) -# -# os.remove(iJob.Launcher22) -# - def test_clean(self): - acronym = "test_clean" - f = open("ClusterLauncher22" + acronym + ".py", "w") - f.close() - f = open(acronym + ".o1", "w") - f.close() - f = open(acronym + ".e1", "w") - f.close() - - launcherParameter = LauncherParameter.createParameter(self._jobdb, self._groupid, acronym); - iLauncher2 = Launcher2(launcherParameter) - iLauncher2.clean(acronym) - self.assertFalse(FileUtils.isRessourceExists("ClusterLauncher22" + acronym + ".py")) - - def test_clean_without_acronym(self): - acronym = "" - acronym2 = "toto" - f = open("ClusterLauncher22" + acronym2 + ".py", "w") - f.close() - f = open(acronym2 + ".o1", "w") - f.close() - f = open(acronym2 + ".e1", "w") - f.close() - - launcherParameter = LauncherParameter.createParameter(self._jobdb, self._groupid, acronym); - iLauncher2 = Launcher2(launcherParameter) - iLauncher2.clean(acronym) - self.assertFalse(FileUtils.isRessourceExists("ClusterLauncher22" + acronym2 + ".py")) - - def test_getQueueNameAndResources_queue_no_resource(self): - configQueue = "all.q" - expQueueName = "all.q" - expResources = [] - launcherParameter = LauncherParameter.createParameter(self._jobdb); - iLauncher2 = Launcher2(launcherParameter) - - obsQueueName, obsResources = iLauncher2.getQueueNameAndResources(configQueue) - self.assertEquals(expQueueName, obsQueueName) - self.assertEquals(expResources, obsResources) - -# def test_getQueueNameAndResources_queue_one_resource(self): -# configQueue = "test.q 'test=TRUE'" -# expQueueName = "test.q" -# expResources = ["test=TRUE"] -# iLauncher2 = Launcher2(self._jobdb) -# obsQueueName, obsResources = iLauncher2.getQueueNameAndResources(configQueue) -# self.assertEquals(expQueueName, obsQueueName) -# self.assertEquals(expResources, obsResources) - -# def test_getQueueNameAndResources_queue_two_resources(self): -# configQueue = "big.q 's_data=8G s_cpu=96:00:00'" -# expQueueName = "big.q" -# expResources = ["s_data=8G", "s_cpu=96:00:00"] -# iLauncher2 = Launcher2(self._jobdb) -# obsQueueName, obsResources = iLauncher2.getQueueNameAndResources(configQueue) -# self.assertEquals(expQueueName, obsQueueName) -# self.assertEquals(expResources, obsResources) - -# def test_getQueueNameAndResources_no_queue_no_resource(self): -# configQueue = "" -# expQueueName = "" -# expResources = [] -# iLauncher2 = Launcher2(self._jobdb) -# obsQueueName, obsResources = iLauncher2.getQueueNameAndResources(configQueue) -# self.assertEquals(expQueueName, obsQueueName) -# self.assertEquals(expResources, obsResources) -# -# def test_getQueueNameAndResources_no_queue_one_resource(self): -# configQueue = "s_data=8G" -# expQueueName = "" -# expResources = ["s_data=8G"] -# iLauncher2 = Launcher2(self._jobdb) -# obsQueueName, obsResources = iLauncher2.getQueueNameAndResources(configQueue) -# self.assertEquals(expQueueName, obsQueueName) -# self.assertEquals(expResources, obsResources) -# -# def test_getQueueNameAndResources_no_queue_two_resource(self): -# configQueue = "s_data=8G s_cpu=96:00:00" -# expQueueName = "" -# expResources = ["s_data=8G", "s_cpu=96:00:00"] -# iLauncher2 = Launcher2(self._jobdb) -# obsQueueName, obsResources = iLauncher2.getQueueNameAndResources(configQueue) -# self.assertEquals(expQueueName, obsQueueName) -# self.assertEquals(expResources, obsResources) -# -## #TODO: test with at least 2 lines in cmd -# def test_runSingleJob(self): -# acronym = "Test_runSingleJob" -# os.mkdir(acronym) -# os.chdir(acronym) -# iLauncher2 = Launcher2(self._jobdb, os.getcwd(), "", "", os.getcwd(), self._tmpDir, "", self._queue, self._groupid, acronym) -# iLauncher2.job.groupid = self._groupid -# iLauncher2.job.jobname = acronym -# iLauncher2.job.queue = self._queue -# if Test_Launcher2.SARUMAN_NAME == os.getenv("HOSTNAME"): -# iLauncher2.job.lResources = ["test=TRUE"] -# cmd = "log = os.system(\"touch 'YuFei'\")\n" -# iLauncher2.runSingleJob(cmd) -# time.sleep(20) -# jobStatus = self._jobdb.getJobStatus(iLauncher2.job) -# os.chdir(self._cDir) -# shutil.rmtree(acronym) -# self.assertEqual(jobStatus, "finished") -# -# def test_runSingleJob_catch_error_wrong_tmpDir(self): -# acronym = "Test_runSingleJob_catch_error" -# os.mkdir(acronym) -# os.chdir(acronym) -# iLauncher2= Launcher2(self._jobdb, os.getcwd(), "", "", os.getcwd(), "%s/toto" % self._tmpDir, "", self._queue, self._groupid, acronym) -# iLauncher2.job.groupid = self._groupid -# iLauncher2.job.jobname = acronym -# iLauncher2.job.queue = self._queue -# if Test_Launcher2.SARUMAN_NAME == os.getenv("HOSTNAME"): -# iLauncher2.job.lResources = ["test=TRUE"] -# cmd = "log = os.system(\"touch 'YuFei'\")\n" -# iLauncher2.runSingleJob(cmd) -# time.sleep(20) -# jobStatus = self._jobdb.getJobStatus(iLauncher2.job) -# os.chdir(self._cDir) -# shutil.rmtree(acronym) -# self.assertEqual(jobStatus, "error") -# -# def test_runSingleJob_catch_error_wrong_cmd(self): -# acronym = "Test_runSingleJob_catch_error" -# os.mkdir(acronym) -# os.chdir(acronym) -# iLauncher2 = Launcher2(self._jobdb, os.getcwd(), "", "", os.getcwd(), self._tmpDir, "", self._queue, self._groupid, acronym) -# iLauncher2.job.groupid = self._groupid -# iLauncher2.job.jobname = acronym -# iLauncher2.job.queue = self._queue -# if Test_Launcher2.SARUMAN_NAME == os.getenv("HOSTNAME"): -# iLauncher2.job.lResources = ["test=TRUE"] -# cmd = "log = os.system(\"truc -i toto\")\n" -# iLauncher2.runSingleJob(cmd) -# time.sleep(20) -# jobStatus = self._jobdb.getJobStatus(iLauncher2.job) -# self._jobdb.cleanJobGroup(self._groupid) -# os.chdir(self._cDir) -# shutil.rmtree(acronym) -# self.assertEqual(jobStatus, "error") -# -# def test_prepareCommands(self): -# expCmdStart = "os.symlink(\"../Yufei_chunks.fa\", \"Yufei_chunks.fa\")\n\tos.symlink(\"../Yufei_chunks.fa_cut\", \"Yufei_chunks.fa_cut\")\n\tlog = os.system(\"touch file\")\n\t" -# expCmdFinish = "if os.path.exists(\"yufei.align\"):\n\t\tshutil.move(\"yufei.align\", \"yufeiLuo/.\" )\n\t" -# expCmdSize = "fileSize = 3.2\n\t\t" -# expCmdCopy = "shutil.copy(\"PY/Yufei_db/Yufei_chunks.fa\", \".\")\n\t\tshutil.copy(\"PY/Yufei_db/Yufei_chunks.fa_cut\", \".\")\n\t\t" -# -# lCmdStart = [] -# lCmdStart.append("os.symlink(\"../Yufei_chunks.fa\", \"Yufei_chunks.fa\")") -# lCmdStart.append("os.symlink(\"../Yufei_chunks.fa_cut\", \"Yufei_chunks.fa_cut\")") -# lCmds = [] -# lCmds.append("log = os.system(\"touch file\")") -# lCmdFinish = [] -# lCmdFinish.append("if os.path.exists(\"yufei.align\"):") -# lCmdFinish.append("\tshutil.move(\"yufei.align\", \"yufeiLuo/.\" )") -# lCmdSize = [] -# lCmdSize.append("fileSize = 3.2") -# lCmdCopy = [] -# lCmdCopy.append("shutil.copy(\"PY/Yufei_db/Yufei_chunks.fa\", \".\")") -# lCmdCopy.append("shutil.copy(\"PY/Yufei_db/Yufei_chunks.fa_cut\", \".\")") -# -# iLauncher2 = Launcher2(self._jobdb) -# obsCmdStart, obsCmdFinish, obsCmdSize, obsCmdCopy = iLauncher2.prepareCommands(lCmds, lCmdStart, lCmdFinish, lCmdSize, lCmdCopy) -# -# self.assertEquals(expCmdStart, obsCmdStart) -# self.assertEquals(expCmdFinish, obsCmdFinish) -# self.assertEquals(expCmdSize, obsCmdSize) -# self.assertEquals(expCmdCopy, obsCmdCopy) -# -# def test_getSystemCommand(self): -# prg = "touch" -# lArgs = [] -# lArgs.append("file") -# expCmd = "log = os.system(\"touch file\")" -# iLauncher2 = Launcher2(self._jobdb) -# obsCmd = iLauncher2.getSystemCommand(prg, lArgs) -# self.assertEquals(expCmd, obsCmd) - - -test_suite = unittest.TestSuite() -test_suite.addTest( unittest.makeSuite( Test_Launcher2 ) ) -if __name__ == "__main__": - unittest.TextTestRunner(verbosity=2).run( test_suite )
--- a/commons/core/launcher/test/Test_LauncherUtils.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,102 +0,0 @@ -import unittest -from commons.core.launcher.LauncherUtils import LauncherUtils - -class Test_LauncherUtils(unittest.TestCase): - - def test_createHomogeneousSizeList_empty(self): - lHeadersSizeTuples = [] - maxSize = 500 - expLHeadersList = [] - obsLHeadersList = LauncherUtils.createHomogeneousSizeList(lHeadersSizeTuples, maxSize) - self.assertEquals(expLHeadersList, obsLHeadersList) - - def test_createHomogeneousSizeList_one_item_upper_mean(self): - lHeadersSizeTuples = [("h1", 300)] - maxSize = 500 - expLHeadersList = [["h1"]] - obsLHeadersList = LauncherUtils.createHomogeneousSizeList(lHeadersSizeTuples, maxSize) - self.assertEquals(expLHeadersList, obsLHeadersList) - - def test_createHomogeneousSizeList_one_item_under_mean(self): - lHeadersSizeTuples = [("h1", 100)] - maxSize = 500 - expLHeadersList = [["h1"]] - obsLHeadersList = LauncherUtils.createHomogeneousSizeList(lHeadersSizeTuples, maxSize) - self.assertEquals(expLHeadersList, obsLHeadersList) - - def test_createHomogeneousSizeList_3items(self): - lHeadersSizeTuples = [("h1", 250), - ("h2", 250), - ("h3", 300)] - maxSize = 500 - expLHeadersList = [["h3"], ["h2"], ["h1"]] - obsLHeadersList = LauncherUtils.createHomogeneousSizeList(lHeadersSizeTuples, maxSize) - self.assertEquals(expLHeadersList, obsLHeadersList) - - def test_createHomogeneousSizeList_4items(self): - lHeadersSizeTuples = [("h1", 100), - ("h2", 200), - ("h3", 10), - ("h4", 400)] - maxSize = 500 - expLHeadersList = [["h4", "h3"], ["h2", "h1"]] - obsLHeadersList = LauncherUtils.createHomogeneousSizeList(lHeadersSizeTuples, maxSize) - self.assertEquals(expLHeadersList, obsLHeadersList) - - def test_createHomogeneousSizeList_5items(self): - lHeadersSizeTuples = [("h1", 300), - ("h2", 300), - ("h3", 250), - ("h4", 100), - ("h5", 90)] - maxSize = 500 - expLHeadersList = [["h2", "h5","h4"], ["h1"], ["h3"]] - obsLHeadersList = LauncherUtils.createHomogeneousSizeList(lHeadersSizeTuples, maxSize) - self.assertEquals(expLHeadersList, obsLHeadersList) - - def test_createHomogeneousSizeList_all_upper_max(self): - lHeadersSizeTuples = [("h1", 600), - ("h2", 500), - ("h3", 700), - ("h4", 900), - ("h5", 500)] - maxSize = 500 - expLHeadersList = [["h4"], ["h3"], ["h1"], ["h5"], ["h2"]] - obsLHeadersList = LauncherUtils.createHomogeneousSizeList(lHeadersSizeTuples, maxSize) - self.assertEquals(expLHeadersList, obsLHeadersList) - - def test_createHomogeneousSizeList_all_upper_mean(self): - lHeadersSizeTuples = [("h1", 300), - ("h2", 300), - ("h3", 300), - ("h4", 300), - ("h5", 300)] - maxSize = 500 - expLHeadersList = [["h5"], ["h4"], ["h3"], ["h2"], ["h1"]] - obsLHeadersList = LauncherUtils.createHomogeneousSizeList(lHeadersSizeTuples, maxSize) - self.assertEquals(expLHeadersList, obsLHeadersList) - - def test_createHomogeneousSizeList_all_under_mean(self): - lHeadersSizeTuples = [("h1", 100), - ("h2", 100), - ("h3", 100), - ("h4", 100), - ("h5", 100)] - maxSize = 500 - expLHeadersList = [["h5", "h4", "h3", "h2"], ["h1"]] - obsLHeadersList = LauncherUtils.createHomogeneousSizeList(lHeadersSizeTuples, maxSize) - self.assertEquals(expLHeadersList, obsLHeadersList) - - def test_createHomogeneousSizeList_floats(self): - lHeadersSizeTuples = [("h1", 99.1), - ("h2", 100.7), - ("h3", 100.1), - ("h4", 100.1), - ("h5", 100)] - maxSize = 500 - expLHeadersList = [['h2', 'h4', 'h3', 'h5'], ["h1"]] - obsLHeadersList = LauncherUtils.createHomogeneousSizeList(lHeadersSizeTuples, maxSize) - self.assertEquals(expLHeadersList, obsLHeadersList) - -if __name__ == "__main__": - unittest.main() \ No newline at end of file
--- a/commons/core/launcher/test/Test_WriteScript.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,365 +0,0 @@ -from commons.core.utils.FileUtils import FileUtils -from commons.core.launcher.WriteScript import WriteScript -from commons.core.sql.Job import Job -from commons.core.sql.DbFactory import DbFactory -from commons.core.sql.TableJobAdaptatorFactory import TableJobAdaptatorFactory -import unittest -import os -import shutil -import time -import threading - -class Test_WriteScript(unittest.TestCase): - - def setUp(self): - self._testDir = os.getcwd() - self._acronym = "dummyAcronym" - self._jobTable = "dummyJobsTable" - self._iDb = DbFactory.createInstance() - self._iDb.createTable(self._jobTable, "jobs", overwrite = True) - self._jobdb = TableJobAdaptatorFactory.createInstance(self._iDb, self._jobTable) - self._job = Job() - self._job.groupid = "groupid" - self._job.jobname = self._acronym - self._job.launcher = "ClusterLauncher" - self._jobdb.recordJob(self._job) - self._dummyScratch = "dummyScratch" - os.mkdir(self._dummyScratch) - os.chdir(self._dummyScratch) - self._tmpDir = os.getcwd() - self._iScriptWriter = WriteScript(self._job, self._jobdb, self._testDir, self._tmpDir) - - def tearDown(self): - self._iDb.dropTable(self._jobTable) - self._iDb.close() - if FileUtils.isRessourceExists(self._dummyScratch): - shutil.rmtree(self._dummyScratch) - - def test_run(self): - isScriptAsRun = False - fileToCreate = 'dummyFile' - cmdStart = "log = os.system( \"touch %s\" )\n" % fileToCreate - cmdFinish = "os.system(\"mv %s %s\" )\n" % (fileToCreate, self._testDir) - pyFileName = "%s/ClusterLauncher_%s.py" % (os.getcwd(), self._acronym) - - self._iScriptWriter.run(cmdStart, cmdFinish, pyFileName) - os.system("python %s" % pyFileName) - - os.chdir(self._testDir) - if FileUtils.isRessourceExists(fileToCreate): - os.remove(fileToCreate) - isScriptAsRun = True - expJobStatus = "finished" - obsJobStatus = self._jobdb.getJobStatus(self._job) - - self.assertTrue(isScriptAsRun) - self.assertEquals(expJobStatus, obsJobStatus) - - def test_run_with_cmdSize_and_cmdCopy(self): - isScriptAsRun = False - fileToCreate = 'dummyFile' - fileSize = 0.5 - cmdSize = "fileSize = %f\n" % fileSize - cmdCopy = "os.system(\"touch bank.fa\")\n" - cmdStart = "log = os.system(\"touch %s\")\n" % fileToCreate - cmdFinish = "shutil.move(\"%s\", \"%s\")" % (fileToCreate, self._testDir) - pyFileName = "%s/ClusterLauncher_%s.py" % (os.getcwd(), self._acronym) - - iWriteScript = WriteScript(self._job, self._jobdb, self._testDir, self._tmpDir, True) - iWriteScript.run(cmdStart, cmdFinish, pyFileName, cmdSize, cmdCopy) - os.system("python %s" % pyFileName) - - os.chdir(self._testDir) - if FileUtils.isRessourceExists(fileToCreate): - os.remove(fileToCreate) - isScriptAsRun = True - expJobStatus = "finished" - obsJobStatus = self._jobdb.getJobStatus(self._job) - - self.assertTrue(isScriptAsRun) - self.assertEquals(expJobStatus, obsJobStatus) - -#TODO: how to test ? -# def test_run_2_jobs_trying_to_create_same_groupIdDir(self): -# fileToCreate1 = 'dummyFile1' -# fileToCreate2 = 'dummyFile2' -# flagFileOSError = "osErrorRaised" -# -# fileSize = 0.5 -# cmd_checkSize = "" -# cmd_checkSize += "if not os.path.exists( \"%s\" ):\n" % self._job.groupid -# cmd_checkSize += "\tfileSize = %f\n" % fileSize -# -# cmd_checkGroupidDir1 = "" -# cmd_checkGroupidDir1 += "if not os.path.exists(\"%s\"):\n" % self._job.groupid -# cmd_checkGroupidDir1 += "\ttry:\n" -# cmd_checkGroupidDir1 += "\t\ttime.sleep(10)\n" -# cmd_checkGroupidDir1 += "\t\tos.mkdir(\"%s\")\n" % self._job.groupid -# cmd_checkGroupidDir1 += "\texcept OSError, e :\n" -# cmd_checkGroupidDir1 += "\t\tos.system(\"touch %s\")\n" % flagFileOSError -# cmd_checkGroupidDir1 += "\t\tif e.args[0] != 17:\n" -# cmd_checkGroupidDir1 += "\t\t\traise RepetException(\"ERROR: can't create '%s'\")\n" % self._job.groupid -# cmd_checkGroupidDir1 += "\tos.chdir(\"%s\")\n" % self._job.groupid -# cmd_checkGroupidDir1 += "\tos.system(\"touch bank.fa\")\n" #cp -# cmd_checkGroupidDir1 += "else:\n" -# cmd_checkGroupidDir1 += "\tos.chdir(\"%s\")\n" % self._job.groupid -# -# cmdStart1 = "log = os.system(\"touch %s\")\n" % fileToCreate1 -# cmdFinish1 = "shutil.move(\"%s\", \"%s\")\n" % (fileToCreate1, self._testDir) -# pyFileName1 = "%s/ClusterLauncher1_job1.py" % os.getcwd() -# -# cmd_checkGroupidDir2 = "" -# cmd_checkGroupidDir2 += "if not os.path.exists(\"%s\"):\n" % self._job.groupid -# cmd_checkGroupidDir2 += "\ttry:\n" -# cmd_checkGroupidDir2 += "\t\tos.mkdir(\"%s\")\n" % self._job.groupid -# cmd_checkGroupidDir2 += "\texcept OSError, e :\n" -# cmd_checkGroupidDir2 += "\t\tif e.args[0] != 17:\n" -# cmd_checkGroupidDir2 += "\t\t\traise RepetException(\"ERROR: can't create '%s'\")\n" % self._job.groupid -# cmd_checkGroupidDir2 += "\tos.chdir(\"%s\")\n" % self._job.groupid -# cmd_checkGroupidDir2 += "\tos.system(\"touch bank.fa\")\n" #cp -# cmd_checkGroupidDir2 += "else:\n" -# cmd_checkGroupidDir2 += "\tos.chdir(\"%s\")\n" % self._job.groupid -# -# cmdStart2 = "log = os.system(\"touch %s\")\n" % fileToCreate2 -# cmdFinish2 = "shutil.move(\"%s\", \"%s\")\n" % (fileToCreate2, self._testDir) -# pyFileName2 = "%s/ClusterLauncher2_job2.py" % os.getcwd() -# -# job1 = Job(self._jobTable, jobname = "job1", groupid = self._job.groupid) -# self._jobdb.recordJob(job1) -# job2 = Job(self._jobTable, jobname = "job2", groupid = self._job.groupid) -# self._jobdb.recordJob(job2) -# iScriptWriter1 = WriteScript(job1, self._jobdb, self._testDir, self._tmpDir) -# iScriptWriter1.run(cmdStart1, cmdFinish1, pyFileName1, cmd_checkSize, cmd_checkGroupidDir1) -# iScriptWriter2 = WriteScript(job2, self._jobdb, self._testDir, self._tmpDir) -# iScriptWriter2.run(cmdStart2, cmdFinish2, pyFileName2, cmd_checkSize, cmd_checkGroupidDir2) -# -# iCFT1 = CreateFileThread(pyFileName1) -# iCFT2 = CreateFileThread(pyFileName2) -# iCFT1.start() -# iCFT2.start() -# while iCFT1.isAlive() or iCFT2.isAlive(): -# time.sleep(5) -# self.assertTrue(FileUtils.isRessourceExists(flagFileOSError)) -# os.chdir(self._testDir) -# -# if FileUtils.isRessourceExists(fileToCreate1): -# os.remove(fileToCreate1) -# -# if FileUtils.isRessourceExists(fileToCreate2): -# os.remove(fileToCreate2) - - def test_run_2_lines_in_cmd_start(self): - isScriptAsRun = False - fileToCreate = 'dummyFile' - - cmdStart = "log = 0\n\t" - cmdStart += "if True:\n\t" - cmdStart += "\tos.system( \"touch dummyFile\" )\n" - cmdFinish = "os.system(\"mv %s %s\" )\n" % (fileToCreate, self._testDir) - pyFileName = "%s/ClusterLauncher_%s.py" % (os.getcwd(), self._acronym) - - self._iScriptWriter.run(cmdStart, cmdFinish, pyFileName) - os.system("python %s" % pyFileName) - - os.chdir(self._testDir) - if FileUtils.isRessourceExists(fileToCreate): - os.remove(fileToCreate) - isScriptAsRun = True - self.assertTrue(isScriptAsRun) - - def test_run_2_lines_in_cmd_finish(self): - isScriptAsRun = False - fileToCreate = 'dummyFile' - - cmdStart = "log = 0\n\t" - cmdStart += "if True:\n\t" - cmdStart += "\tos.system( \"touch dummyFile\" )\n" - cmdFinish = "if True:\n\t" - cmdFinish += "\tos.system(\"mv %s %s\" )\n" % (fileToCreate, self._testDir) - pyFileName = "%s/ClusterLauncher_%s.py" % (os.getcwd(), self._acronym) - - self._iScriptWriter.run(cmdStart, cmdFinish, pyFileName) - os.system("python %s" % pyFileName) - - os.chdir(self._testDir) - if FileUtils.isRessourceExists(fileToCreate): - os.remove(fileToCreate) - isScriptAsRun = True - self.assertTrue(isScriptAsRun) - - def test_fillTemplate_with_JobScriptTemplate(self): - os.chdir("..") - d = { - "tmpDir" : "/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch", - "jobTableName" : "dummyJobsTable", - "groupId" : "groupid", - "jobName" : "job1", - "launcher" : "ClusterLauncher", - "time" : "20110505-105353", - "repet_path" : "/home/user/workspace/repet_pipe", - "repet_host" : "pisano", - "repet_user" : "user", - "repet_pw" : "user", - "repet_db" : "repet_user", - "repet_port" : "3306", - "cmdStart" : "log = os.system(\"touch dummyFile1\")", - "cmdFinish" : "shutil.move(\"dummyFile1\", \"/home/user/workspace/repet_pipe/commons/core/launcher/test\")", - "cDir" : "/home/user/workspace/repet_pipe/commons/core/launcher/test/" - } - expFileName = "expFiles/expJobScriptTemplate.py" - obsFileName = "obsFile.py" - - iWS = WriteScript() - iWS.fillTemplate(obsFileName, d) - self.assertTrue(FileUtils.are2FilesIdentical(expFileName, obsFileName)) - os.remove(obsFileName) - - def test_fillTemplate_with_JobScriptTemplate_2_lines_in_cmd_start(self): - os.chdir("..") - d = { - "tmpDir" : "/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch", - "jobTableName" : "dummyJobsTable", - "groupId" : "groupid", - "jobName" : "job1", - "launcher" : "ClusterLauncher", - "time" : "20110505-105353", - "repet_path" : "/home/user/workspace/repet_pipe", - "repet_host" : "pisano", - "repet_user" : "user", - "repet_pw" : "user", - "repet_db" : "repet_user", - "repet_port" : "3306", - "cmdStart" : "print \"Hello Yufei\"\n\tlog = os.system(\"touch dummyFile1\")", - "cmdFinish" : "shutil.move(\"dummyFile1\", \"/home/user/workspace/repet_pipe/commons/core/launcher/test\")", - "cDir" : "/home/user/workspace/repet_pipe/commons/core/launcher/test/" - } - expFileName = "expFiles/expJobScriptTemplate_cmdWith2Lines.py" - obsFileName = "obsFile.py" - - iWS = WriteScript() - iWS.fillTemplate(obsFileName, d) - self.assertTrue(FileUtils.are2FilesIdentical(expFileName, obsFileName)) - os.remove(obsFileName) - - def test_fillTemplate_with_JobScriptWithFilesCopyTemplate(self): - os.chdir("..") - d = { - "tmpDir" : "/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch", - "jobTableName" : "dummyJobsTable", - "groupId" : "groupid", - "jobName" : "job1", - "launcher" : "ClusterLauncher", - "time" : "20110505-105353", - "repet_path" : "/home/user/workspace/repet_pipe", - "repet_host" : "pisano", - "repet_user" : "user", - "repet_pw" : "user", - "repet_db" : "repet_user", - "repet_port" : "3306", - "cmdStart" : "log = os.system(\"touch dummyFile1\")", - "cmdFinish" : "shutil.move(\"dummyFile1\", \"/home/user/workspace/repet_pipe/commons/core/launcher/test\")", - "cDir" : "/home/user/workspace/repet_pipe/commons/core/launcher/test/", - "cmdSize" : "fileSize = 0.500000", - "cmdCopy" : "os.system(\"touch bank.fa\")" - } - expFileName = "expFiles/expJobScriptWithFilesCopyTemplate.py" - obsFileName = "obsFile.py" - - iWS = WriteScript(chooseTemplateWithCopy = True) - iWS.fillTemplate(obsFileName, d) - self.assertTrue(FileUtils.are2FilesIdentical(expFileName, obsFileName)) - os.remove(obsFileName) - - def test_fillTemplate_with_JobScriptTemplateLight(self): - os.chdir("..") - d = { - "tmpDir" : "/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch", - "jobTableName" : "dummyJobsTable", - "groupId" : "groupid", - "jobName" : "job1", - "launcher" : "ClusterLauncher", - "time" : "20110505-105353", - "repet_path" : "/home/user/workspace/repet_pipe", - "cmdStart" : "log = os.system(\"touch dummyFile1\")", - "cmdFinish" : "shutil.move(\"dummyFile1\", \"/home/user/workspace/repet_pipe/commons/core/launcher/test\")", - "cDir" : "/home/user/workspace/repet_pipe/commons/core/launcher/test/", - "cmdSize" : "fileSize = 0.500000", - "cmdCopy" : "os.system(\"touch bank.fa\")" - } - expFileName = "expFiles/expJobScriptTemplateLight.py" - obsFileName = "obs.py" - - iWS = WriteScript(chooseTemplateLight = True) - iWS.fillTemplate(obsFileName, d) - self.assertTrue(FileUtils.are2FilesIdentical(expFileName, obsFileName)) - os.remove(obsFileName) - - def test_createJobScriptDict(self): - os.chdir("..") - cmd_start = "log = os.system(\"touch dummyFile1\")" - cmd_finish = "shutil.move(\"dummyFile1\", \"/home/user/workspace/repet_pipe/commons/core/launcher/test\")" - cmd_size = "" - cmd_copy = "" - expDict = { - "tmpDir" : self._tmpDir, - "jobTableName" : self._jobTable, - "groupId" : self._job.groupid, - "jobName" : self._acronym, - "launcher" : self._job.launcher, - "time" : time.strftime("%Y%m%d-%H%M%S"), - "repet_path" : os.environ["REPET_PATH"], - "repet_host" : os.environ["REPET_HOST"], - "repet_user" : os.environ["REPET_USER"], - "repet_pw" : os.environ["REPET_PW"], - "repet_db" : os.environ["REPET_DB"], - "repet_port" : os.environ["REPET_PORT"], - "cmdStart" : cmd_start, - "cmdFinish" : cmd_finish, - "cDir" : self._testDir, - "cmdSize" : cmd_size, - "cmdCopy" : cmd_copy - } - obsDict = self._iScriptWriter.createJobScriptDict(cmd_start, cmd_finish, cmd_size, cmd_copy) - self.assertEquals(expDict, obsDict) - - def test_createJobScriptDict_with_cmdSize_and_cmdCopy(self): - os.chdir("..") - cmd_start = "log = os.system(\"touch dummyFile1\")" - cmd_finish = "shutil.move(\"dummyFile1\", \"/home/user/workspace/repet_pipe/commons/core/launcher/test\")" - cmd_size = "fileSize = 0.500000" - cmd_copy = "os.system(\"touch bank.fa\")" - expDict = { - "tmpDir" : self._tmpDir, - "jobTableName" : self._jobTable, - "groupId" : self._job.groupid, - "jobName" : self._acronym, - "launcher" : self._job.launcher, - "time" : time.strftime("%Y%m%d-%H%M%S"), - "repet_path" : os.environ["REPET_PATH"], - "repet_host" : os.environ["REPET_HOST"], - "repet_user" : os.environ["REPET_USER"], - "repet_pw" : os.environ["REPET_PW"], - "repet_db" : os.environ["REPET_DB"], - "repet_port" : os.environ["REPET_PORT"], - "cmdStart" : cmd_start, - "cmdFinish" : cmd_finish, - "cDir" : self._testDir, - "cmdSize" : cmd_size, - "cmdCopy" : cmd_copy - } - obsDict = self._iScriptWriter.createJobScriptDict(cmd_start, cmd_finish, cmd_size, cmd_copy) - self.assertEquals(expDict, obsDict) - -class CreateFileThread(threading.Thread): - - def __init__(self, pyFileName): - threading.Thread.__init__(self) - self._pyFileName = pyFileName - - def run(self): - os.system("python %s" % self._pyFileName) - -test_suite = unittest.TestSuite() -test_suite.addTest( unittest.makeSuite( Test_WriteScript ) ) -if __name__ == "__main__": - unittest.TextTestRunner(verbosity=2).run( test_suite )
--- a/commons/core/launcher/test/expFiles/expJobScriptSQLiteWithFilesCopyTemplate.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,107 +0,0 @@ -#!/usr/bin/env python - -import os -import sys -import time -import shutil -from commons.core.checker.RepetException import RepetException -from commons.core.sql.TableJobAdaptator import TableJobAdaptator -from commons.core.sql.DbMySql import DbMySql -from commons.core.sql.DbSQLite import DbSQLite -from commons.core.sql.Job import Job - -try: - newDir = None - print os.uname() - beginTime = time.time() - print 'beginTime=%f' % beginTime - print "work in dir '/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch'" - sys.stdout.flush() - if not os.path.exists("/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch"): - raise IOError("ERROR: temporary directory '/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch' doesn't exist") - - fileSize = 0 - if not os.path.exists("groupid"): - fileSize = 0.500000 - freeGigaNeededInTmpDir = float(1 + fileSize) - freeSpace = os.statvfs("/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch") - if ((freeSpace.f_bavail * freeSpace.f_frsize) / 1073741824.0 < freeGigaNeededInTmpDir): - raise RepetException("ERROR: less than %.2fG of input file in '/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch'" % freeGigaNeededInTmpDir) - - os.chdir("/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch") - if not os.path.exists("groupid"): - try: - os.mkdir("groupid") - except OSError, e : - if e.args[0] != 17: - raise RepetException("ERROR: can't create 'groupid'") - os.chdir("groupid") - os.system("touch bank.fa") - else: - os.chdir("groupid") - - newDir = "groupid_job1_20110505-105353" - if os.path.exists(newDir): - shutil.rmtree(newDir) - os.mkdir(newDir) - os.chdir(newDir) - - queue = "main.q" - iJob = Job("jobs", jobname = "job1", groupid = "groupid", queue = queue, node = os.getenv("HOSTNAME")) - iDb = DbSQLite("/home/user/workspace/repet_pipe/commons/core/launcher/test/jobs") - iTJA = TableJobAdaptator(iDb, "jobs") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "running") - print "updated status: %s" % iTJA.getJobStatus(iJob) - iDb.close() - - log = os.system("touch dummyFile1") - if log != 0: - raise RepetException("ERROR: job returned %i" % log) - else: - print "job finished successfully" - shutil.move("dummyFile1", "/home/user/workspace/repet_pipe/commons/core/launcher/test") - - os.chdir("..") - shutil.rmtree(newDir) - - iDb = DbSQLite("/home/user/workspace/repet_pipe/commons/core/launcher/test/jobs") - iTJA = TableJobAdaptator(iDb, "jobs") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "finished") - print "updated status: %s" % iTJA.getJobStatus(iJob) - iDb.close() - - endTime = time.time() - print 'endTime=%f' % endTime - print 'executionTime=%f' % (endTime - beginTime) - print os.uname() - -except IOError, e : - print e - queue = "main.q" - iJob = Job("jobs", jobname = "job1", groupid = "groupid", queue = queue, node = os.getenv("HOSTNAME")) - iDb = DbSQLite("/home/user/workspace/repet_pipe/commons/core/launcher/test/jobs") - iTJA = TableJobAdaptator(iDb, "jobs") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "error") - print "updated status: %s" % iTJA.getJobStatus(iJob) - iDb.close() - sys.exit(1) - -except Exception, e : - print "tmpDir is : /home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch" - print "cDir is : /home/user/workspace/repet_pipe/commons/core/launcher/test/" - print e - if newDir != None and os.path.exists("../%s" % newDir) and not os.path.exists("/home/user/workspace/repet_pipe/commons/core/launcher/test//%s" % newDir): - os.chdir("..") - shutil.move(newDir, "/home/user/workspace/repet_pipe/commons/core/launcher/test//%s" % newDir) - queue = "main.q" - iJob = Job("jobs", jobname = "job1", groupid = "groupid", queue = queue, node = os.getenv("HOSTNAME")) - iDb = DbSQLite("/home/user/workspace/repet_pipe/commons/core/launcher/test/jobs") - iTJA = TableJobAdaptator(iDb, "jobs") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "error") - print "updated status: %s" % iTJA.getJobStatus(iJob) - iDb.close() - sys.exit(1)
--- a/commons/core/launcher/test/expFiles/expJobScriptTemplate.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,95 +0,0 @@ -#!/usr/bin/env python - -import os -import sys -import time -import shutil -from commons.core.checker.RepetException import RepetException -from commons.core.sql.TableJobAdaptator import TableJobAdaptator -from commons.core.sql.DbFactory import DbFactory -from commons.core.sql.Job import Job - -try: - newDir = None - print os.uname() - beginTime = time.time() - print 'beginTime=%f' % beginTime - print "work in dir '/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch'" - sys.stdout.flush() - if not os.path.exists( "/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch" ): - raise IOError("ERROR: temporary directory '/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch' doesn't exist") - - minFreeGigaInTmpDir = 1 - freeSpace = os.statvfs("/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch") - if ((freeSpace.f_bavail * freeSpace.f_frsize) / 1073741824.0 < minFreeGigaInTmpDir): - raise RepetException("ERROR: less than %iG of free space in '/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch'" % minFreeGigaInTmpDir) - - os.chdir("/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch") - newDir = "groupid_job1_20110505-105353" - if os.path.exists(newDir): - shutil.rmtree(newDir) - os.mkdir(newDir) - os.chdir(newDir) - - iJob = Job(jobname = "job1", groupid = "groupid", launcherFile = "ClusterLauncher", node = os.getenv("HOSTNAME")) - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "dummyJobsTable") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "running") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - - log = os.system("touch dummyFile1") - if log != 0: - raise RepetException("ERROR: job returned %i" % log) - else: - print "job finished successfully" - sys.stdout.flush() - shutil.move("dummyFile1", "/home/user/workspace/repet_pipe/commons/core/launcher/test") - - os.chdir("..") - shutil.rmtree(newDir) - - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "dummyJobsTable") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "finished") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - - endTime = time.time() - print 'endTime=%f' % endTime - print 'executionTime=%f' % (endTime - beginTime) - print os.uname() - sys.stdout.flush() - -except IOError, e : - print e - iJob = Job(jobname = "job1", groupid = "groupid", launcherFile = "ClusterLauncher", node = os.getenv("HOSTNAME")) - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "dummyJobsTable") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "error") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - sys.exit(1) - -except Exception, e : - print "tmpDir is : /home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch" - print "cDir is : /home/user/workspace/repet_pipe/commons/core/launcher/test/" - print e - if newDir != None and os.path.exists("../%s" % newDir) and not os.path.exists("/home/user/workspace/repet_pipe/commons/core/launcher/test//%s" % newDir): - os.chdir("..") - shutil.move(newDir, "/home/user/workspace/repet_pipe/commons/core/launcher/test//%s" % newDir) - iJob = Job(jobname = "job1", groupid = "groupid", launcherFile = "ClusterLauncher", node = os.getenv("HOSTNAME")) - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "dummyJobsTable") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "error") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - sys.exit(1)
--- a/commons/core/launcher/test/expFiles/expJobScriptTemplateLight.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,49 +0,0 @@ -#!/usr/bin/env python - -import os -import sys -import time -import shutil -from commons.core.checker.RepetException import RepetException -try: - newDir = None - print os.uname() - beginTime = time.time() - print 'beginTime=%f' % beginTime - print "work in dir '/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch'" - sys.stdout.flush() - if not os.path.exists( "/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch" ): - raise IOError("ERROR: temporary directory '/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch' doesn't exist") - - minFreeGigaInTmpDir = 1 - freeSpace = os.statvfs("/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch") - if ((freeSpace.f_bavail * freeSpace.f_frsize) / 1073741824.0 < minFreeGigaInTmpDir): - raise RepetException("ERROR: less than %iG of free space in '/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch'" % minFreeGigaInTmpDir) - - os.chdir("/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch") - newDir = "groupid_job1_20110505-105353" - if os.path.exists(newDir): - shutil.rmtree(newDir) - os.mkdir(newDir) - os.chdir(newDir) - - log = os.system("touch dummyFile1") - if log != 0: - raise RepetException("ERROR: job returned %i" % log) - else: - print "job finished successfully" - sys.stdout.flush() - shutil.move("dummyFile1", "/home/user/workspace/repet_pipe/commons/core/launcher/test") - - os.chdir("..") - shutil.rmtree(newDir) - endTime = time.time() - print 'endTime=%f' % endTime - print 'executionTime=%f' % (endTime - beginTime) - print os.uname() - sys.stdout.flush() - -except IOError, e : - print e - sys.stdout.flush() - sys.exit(1) \ No newline at end of file
--- a/commons/core/launcher/test/expFiles/expJobScriptTemplate_cmdWith2Lines.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,96 +0,0 @@ -#!/usr/bin/env python - -import os -import sys -import time -import shutil -from commons.core.checker.RepetException import RepetException -from commons.core.sql.TableJobAdaptator import TableJobAdaptator -from commons.core.sql.DbFactory import DbFactory -from commons.core.sql.Job import Job - -try: - newDir = None - print os.uname() - beginTime = time.time() - print 'beginTime=%f' % beginTime - print "work in dir '/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch'" - sys.stdout.flush() - if not os.path.exists( "/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch" ): - raise IOError("ERROR: temporary directory '/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch' doesn't exist") - - minFreeGigaInTmpDir = 1 - freeSpace = os.statvfs("/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch") - if ((freeSpace.f_bavail * freeSpace.f_frsize) / 1073741824.0 < minFreeGigaInTmpDir): - raise RepetException("ERROR: less than %iG of free space in '/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch'" % minFreeGigaInTmpDir) - - os.chdir("/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch") - newDir = "groupid_job1_20110505-105353" - if os.path.exists(newDir): - shutil.rmtree(newDir) - os.mkdir(newDir) - os.chdir(newDir) - - iJob = Job(jobname = "job1", groupid = "groupid", launcherFile = "ClusterLauncher", node = os.getenv("HOSTNAME")) - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "dummyJobsTable") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "running") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - - print "Hello Yufei" - log = os.system("touch dummyFile1") - if log != 0: - raise RepetException("ERROR: job returned %i" % log) - else: - print "job finished successfully" - sys.stdout.flush() - shutil.move("dummyFile1", "/home/user/workspace/repet_pipe/commons/core/launcher/test") - - os.chdir("..") - shutil.rmtree(newDir) - - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "dummyJobsTable") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "finished") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - - endTime = time.time() - print 'endTime=%f' % endTime - print 'executionTime=%f' % (endTime - beginTime) - print os.uname() - sys.stdout.flush() - -except IOError, e : - print e - iJob = Job(jobname = "job1", groupid = "groupid", launcherFile = "ClusterLauncher", node = os.getenv("HOSTNAME")) - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "dummyJobsTable") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "error") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - sys.exit(1) - -except Exception, e : - print "tmpDir is : /home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch" - print "cDir is : /home/user/workspace/repet_pipe/commons/core/launcher/test/" - print e - if newDir != None and os.path.exists("../%s" % newDir) and not os.path.exists("/home/user/workspace/repet_pipe/commons/core/launcher/test//%s" % newDir): - os.chdir("..") - shutil.move(newDir, "/home/user/workspace/repet_pipe/commons/core/launcher/test//%s" % newDir) - iJob = Job(jobname = "job1", groupid = "groupid", launcherFile = "ClusterLauncher", node = os.getenv("HOSTNAME")) - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "dummyJobsTable") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "error") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - sys.exit(1)
--- a/commons/core/launcher/test/expFiles/expJobScriptWithFilesCopyTemplate.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,109 +0,0 @@ -#!/usr/bin/env python - -import os -import sys -import time -import shutil -from commons.core.checker.RepetException import RepetException -from commons.core.sql.TableJobAdaptator import TableJobAdaptator -from commons.core.sql.DbFactory import DbFactory -from commons.core.sql.Job import Job - -try: - newDir = None - print os.uname() - beginTime = time.time() - print 'beginTime=%f' % beginTime - print "work in dir '/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch'" - sys.stdout.flush() - if not os.path.exists("/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch"): - raise IOError("ERROR: temporary directory '/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch' doesn't exist") - - fileSize = 0 - if not os.path.exists("groupid"): - fileSize = 0.500000 - freeGigaNeededInTmpDir = float(1 + fileSize) - freeSpace = os.statvfs("/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch") - if ((freeSpace.f_bavail * freeSpace.f_frsize) / 1073741824.0 < freeGigaNeededInTmpDir): - raise RepetException("ERROR: less than %.2fG of free space in '/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch'" % freeGigaNeededInTmpDir) - - os.chdir("/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch") - if not os.path.exists("groupid"): - try: - os.mkdir("groupid") - except OSError, e : - if e.args[0] != 17: - raise RepetException("ERROR: can't create 'groupid'") - os.chdir("groupid") - os.system("touch bank.fa") - else: - os.chdir("groupid") - - newDir = "groupid_job1_20110505-105353" - if os.path.exists(newDir): - shutil.rmtree(newDir) - os.mkdir(newDir) - os.chdir(newDir) - - iJob = Job(jobname = "job1", groupid = "groupid", launcherFile = "ClusterLauncher", node = os.getenv("HOSTNAME")) - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "dummyJobsTable") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "running") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - - log = os.system("touch dummyFile1") - if log != 0: - raise RepetException("ERROR: job returned %i" % log) - else: - print "job finished successfully" - sys.stdout.flush() - shutil.move("dummyFile1", "/home/user/workspace/repet_pipe/commons/core/launcher/test") - - os.chdir("..") - shutil.rmtree(newDir) - - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "dummyJobsTable") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "finished") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - - endTime = time.time() - print 'endTime=%f' % endTime - print 'executionTime=%f' % (endTime - beginTime) - print os.uname() - sys.stdout.flush() - -except IOError, e : - print e - iJob = Job(jobname = "job1", groupid = "groupid", launcherFile = "ClusterLauncher", node = os.getenv("HOSTNAME")) - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "dummyJobsTable") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "error") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - sys.exit(1) - -except Exception, e : - print "tmpDir is : /home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch" - print "cDir is : /home/user/workspace/repet_pipe/commons/core/launcher/test/" - print e - if newDir != None and os.path.exists("../%s" % newDir) and not os.path.exists("/home/user/workspace/repet_pipe/commons/core/launcher/test//%s" % newDir): - os.chdir("..") - shutil.move(newDir, "/home/user/workspace/repet_pipe/commons/core/launcher/test//%s" % newDir) - iJob = Job(jobname = "job1", groupid = "groupid", launcherFile = "ClusterLauncher", node = os.getenv("HOSTNAME")) - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "dummyJobsTable") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "error") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - sys.exit(1)
--- a/commons/core/sql/DbFactory.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,38 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - -from commons.core.sql.DbMySql import DbMySql - -class DbFactory (object): - - def createInstance(configFileName = "", verbosity = 1): - return DbMySql(cfgFileName = configFileName, verbosity = verbosity) - - createInstance = staticmethod(createInstance) \ No newline at end of file
--- a/commons/core/sql/DbMySql.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,851 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - -# Exception hierarchy: -# -# StandardError -# |__Warning -# |__Error -# |__InterfaceError -# |__DatabaseError -# |__DataError -# |__OperationalError -# |__IntegrityError -# |__InternalError -# |__ProgrammingError -# |__NotSupportedError - -import os -import sys -import time -import ConfigParser -import MySQLdb -from MySQLdb import InterfaceError -from MySQLdb import OperationalError -from MySQLdb import InternalError -from MySQLdb import DatabaseError -from commons.core.seq.Bioseq import Bioseq -from commons.core.LoggerFactory import LoggerFactory -from commons.core.checker.RepetException import RepetException -from commons.core.sql.TablePathAdaptator import TablePathAdaptator -from commons.core.sql.TableSetAdaptator import TableSetAdaptator - -LOG_DEPTH = "repet.commons" - -TABLE_SCHEMA_DESCRIPTOR = {"map": [("name", "varchar(255)"), ("chr", "varchar(255)"), ("start", "int"), ("end", "int")], - "set": [("path", "int unsigned"), ("name", "varchar(255)"), ("chr", "varchar(255)"), ("start", "int"), ("end", "int")], - "match": [("query_name", "varchar(255)"), ("query_start", "int"), ("query_end", "int"), ("query_length", "int unsigned"), ("query_length_perc", "float"), - ("match_length_perc", "float"), ("subject_name", "varchar(255)"), ("subject_start", "int unsigned"), ("subject_end", "int unsigned"), - ("subject_length", "int unsigned"), ("subject_length_perc", "float"), ("E_value", "double"), ("score", "int unsigned"), ("identity", "float"), - ("path", "int unsigned")], - "path": [("path", "int unsigned"), ("query_name", "varchar(255)"), ("query_start", "int"), ("query_end", "int"), ("subject_name", "varchar(255)"), - ("subject_start", "int unsigned"), ("subject_end", "int unsigned"), ("E_value", "double"), ("score", "int unsigned"), ("identity", "float")], - "align": [("query_name", "varchar(255)"), ("query_start", "int"), ("query_end", "int"), ("subject_name", "varchar(255)"), ("subject_start", "int unsigned"), - ("subject_end", "int unsigned"), ("E_value", "double"), ("score", "int unsigned"), ("identity", "float")], - "seq": [("accession", "varchar(255)"), ("sequence", "longtext"), ("description", "varchar(255)"), ("length", "int unsigned")], - "length": [("accession", "varchar(255)"), ("length", "int unsigned")], - "jobs": [("jobid", "int unsigned"), ("jobname", "varchar(255)"), ("groupid", "varchar(255)"), ("launcher", "varchar(1024)"), - ("queue", "varchar(255)"), ("resources", "varchar(255)"), ("status", "varchar(255)"), ("time", "datetime"), ("node", "varchar(255)")], - "classif": [("seq_name", "varchar(255)"), ("length", "int unsigned"), ("strand", "char"), ("status", "varchar(255)"), ("class_classif", "varchar(255)"), - ("order_classif", "varchar(255)"), ("completeness", "varchar(255)"), ("evidence", "text")], - "pathstat": [("family", "varchar(255)"), ("maxLength", "int"), ("meanLength", "int"), ("covg", "int"), ("frags", "int"), ("fullLgthFrags", "int"), ("copies", "int"), - ("fullLgthCopies", "int"), ("meanId", "varchar(255)"), ("sdId", "varchar(255)"), ("minId", "varchar(255)"), ("q25Id", "varchar(255)"), ("medId", "varchar(255)"), - ("q75Id", "varchar(255)"), ("maxId", "varchar(255)"), ("meanLgth", "varchar(255)"), ("sdLgth", "varchar(255)"), ("minLgth", "varchar(255)"), ("q25Lgth", "varchar(255)"), - ("medLgth", "varchar(255)"), ("q75Lgth", "varchar(255)"), ("maxLgth", "varchar(255)"), ("meanLgthPerc", "varchar(255)"), ("sdLgthPerc", "varchar(255)"), - ("minLgthPerc", "varchar(255)"), ("q25LgthPerc", "varchar(255)"), ("medLgthPerc", "varchar(255)"), ("q75LgthPerc", "varchar(255)"), ("maxLgthPerc", "varchar(255)")], - "info_tables":[("name", "varchar(255)"), ("file", "varchar(255)")] - } - -TABLE_INDEX_DESCRIPTOR = {"map": [("iname", "name"), ("ichr", "chr"), ("istart", "start"), ("iend", "end"), ("icoord", "start, end")], - "set": [("id", "path"), ("iname", "name"), ("ichr", "chr"), ("istart", "start"), ("iend", "end"), ("icoord", "start, end")], - "match": [("id", "path"), ("qname", "query_name"), ("qstart", "query_start"), ("qend", "query_end"), - ("sname", "subject_name"), ("sstart", "subject_start"), ("send", "subject_end"), ("qcoord", "query_start, query_end")], - "path": [("id", "path"), ("qname", "query_name"), ("qstart", "query_start"), ("qend", "query_end"), - ("sname", "subject_name"), ("sstart", "subject_start"), ("send", "subject_end"), ("qcoord", "query_start, query_end")], - "align": [("qname", "query_name"), ("qstart", "query_start"), ("qend", "query_end"), - ("sname", "subject_name"), ("sstart", "subject_start"), ("send", "subject_end"), ("qcoord", "query_start, query_end")], - "seq": [("iacc", "accession"), ("idescr", "description")], - "length": [("iacc", "accession"), ("ilength", "length")], - "jobs": [("ijobid", "jobid"), ("ijobname", "jobname"), ("igroupid", "groupid"), ("istatus", "status")], - "classif": [("iseq_name", "seq_name"), ("istatus", "status"), ("iclass", "class_classif"), ("iorder", "order_classif"), ("icomp", "completeness")], - "pathstat": [], - "info_tables": [] - } - -TABLE_TYPE_SYNONYMS = {"tab": "match", - "fasta": "seq", - "fa": "seq", - "fsa": "seq" - } - -## Handle connections to MySQL tables formatted for REPET -# -class DbMySql(object): - - ## Constructor - # - # @param user string db user name - # @param host string db host name - # @param passwd string db user password - # @param dbname string database name - # @param port integer database port - # @param cfgFileName string configuration file name - # - # @note when a parameter is left blank, the constructor is able - # to set attribute values from environment variables: REPET_HOST, - # REPET_USER, REPET_PW, REPET_DB, REPET_PORT - # - def __init__(self, user = "", host = "", passwd = "", dbname = "", port = "", cfgFileName = "", verbosity = 1): - self._log = LoggerFactory.createLogger("%s.%s" % (LOG_DEPTH, self.__class__.__name__), verbosity) - if cfgFileName != "": - self.setAttributesFromConfigFile(cfgFileName) - - elif host != "" and user != "" and passwd != "" and dbname != "": - self.host = host - self.user = user - self.passwd = passwd - self.dbname = dbname - - else: - for envVar in ["REPET_HOST","REPET_USER","REPET_PW","REPET_DB"]: - if os.environ.get( envVar ) == None: - msg = "ERROR: can't find environment variable '%s'" % envVar - self._log.error(msg) - raise RepetException(msg) - self.host = os.environ.get("REPET_HOST") - self.user = os.environ.get("REPET_USER") - self.passwd = os.environ.get("REPET_PW") - self.dbname = os.environ.get("REPET_DB") - - if port != "" and cfgFileName == "": - self.port = int(port) - elif os.environ.get("REPET_PORT") != None: - self.port = int(os.environ.get("REPET_PORT")) - else: - self.port = 3306 - - maxNbTry = 10 - for i in xrange(1,maxNbTry+1): - if not self.open(): - time.sleep(2) - if i == maxNbTry: - msg = "ERROR: failed to connect to the MySQL database" - self._log.error(msg) - raise DatabaseError(msg) - else: - break - - self.cursor = self.db.cursor() - self.execute("""use %s""" %(self.dbname)) - - - ## Set the attributes from the configuration file - # - # @param configFileName string configuration file name - # - def setAttributesFromConfigFile(self, configFileName): - config = ConfigParser.ConfigParser() - config.readfp( open(configFileName) ) - self.host = config.get("repet_env","repet_host") - self.user = config.get("repet_env","repet_user") - self.passwd = config.get("repet_env","repet_pw") - self.dbname = config.get("repet_env","repet_db") - self.port = int( config.get("repet_env","repet_port") ) - - - ## Connect to the MySQL database - # - def open(self): - try: - if int(MySQLdb.get_client_info().split(".")[0]) >= 5: - self.db = MySQLdb.connect( user = self.user, host = self.host,\ - passwd = self.passwd, db = self.dbname, \ - port = self.port, \ - local_infile = 1 ) - else: - self.db = MySQLdb.connect( user = self.user, host = self.host,\ - passwd = self.passwd, db = self.dbname, \ - port = self.port ) - except MySQLdb.Error, e: - msg = "ERROR %d: %s" % (e.args[0], e.args[1]) - self._log.error(msg) - return False - - return True - - - ## Execute a SQL query - # - # @param qry string SQL query to execute - # @param params parameters of SQL query - # - def execute(self, qry, params = None, nbTry = 3, sleep = 5): - if nbTry: - self._log.debug("################START SQL DEBUG MODE################") - self._log.debug("Current directory: %s" % os.getcwd()) - self._log.debug("Host: %s" % self.host) - self._log.debug("User: %s" % self.user) - self._log.debug("Database: %s" % self.dbname) - self._log.debug("SQL command: %s" % qry) - self._log.debug("################STOP SQL DEBUG MODE################\n") - - try: - if params == None: - self.cursor.execute(qry) - else: - self.cursor.execute(qry, params) - except (InterfaceError, OperationalError, InternalError) as iError: - self._log.error("FAILED to execute query '%s': %s. %s retries left." % (qry, iError.args[1], nbTry - 1)) - self._log.debug("WAIT %is to execute '%s'" % (sleep, qry)) - time.sleep(sleep) - try: - self.close() - except: - pass - self.open() - self.cursor = self.db.cursor() - self.execute(qry, params, nbTry - 1, sleep) - else: - msg = "ERROR: can't execute '%s' after several tries" % qry - self._log.error(msg) - raise DatabaseError(msg) - - ## Close the connection - # - def close( self ): - self.db.close() - - - ## Retrieve the results of a SQL query - # - def fetchall(self): - return self.cursor.fetchall() - - - ## Test if a table exists - # - # @param table string table name - # @return boolean True if the table exists, False otherwise - # - def doesTableExist( self, table ): - self.execute( """SHOW TABLES""" ) - results = self.cursor.fetchall() - if (table,) in results: - return True - return False - - - ## Remove a table if it exists - # - # @param table string table name - # - def dropTable(self, table): - if self.doesTableExist( table ): - sqlCmd = "DROP TABLE %s" % table - self.execute( sqlCmd ) - sqlCmd = 'DELETE FROM info_tables WHERE name = "%s"' % table - self.execute( sqlCmd ) - - - ## Rename a table - # - # @param table string existing table name - # @param newName string new table name - # - def renameTable( self, table, newName ): - self.dropTable( newName ) - self.execute( 'RENAME TABLE %s TO %s ;' % (table, newName) ) - self.execute( 'UPDATE info_tables SET name="%s" WHERE name="%s";' % (newName, table) ) - - - ## Duplicate a table - # - # @param tableName string source table name - # @param newTableName string new table name - # - def copyTable(self, sourcetableName, newTableName): - self.dropTable( newTableName ) - sqlCmd = "CREATE TABLE %s LIKE %s;" % (newTableName, sourcetableName) - self.execute( sqlCmd ) - sqlCmd = "INSERT INTO %s SELECT * FROM %s;" % (newTableName, sourcetableName) - self.execute( sqlCmd ) - self._log.info("copying table data,", sourcetableName, "in", newTableName) - self.updateInfoTable(newTableName, "") - - - ## Give the rows number of the table - # - # @param tableName string table name - # - def getSize( self, tableName ): - qry = "SELECT count(*) FROM %s;" % (tableName) - self.execute(qry) - res = self.fetchall() - return int( res[0][0] ) - - - def getTableType(self, tableName): - qry = "SHOW COLUMNS FROM %s;" % (tableName) - self.execute(qry) - res = self.fetchall() - - fieldNames = [] - for row in res: - fieldNames.append(row[0]) - - for tableType, fieldInfos in TABLE_SCHEMA_DESCRIPTOR.items(): - refFieldsNames = [name for name,type in fieldInfos] - if refFieldsNames == fieldNames: - return tableType - - return None - - - ## Test if table is empty - # - # @param tableName string table name - # @return boolean True if the table is empty, False otherwise - # - def isEmpty(self, tableName): - return self.getSize(tableName) == 0 - - - ## Record a new table in the 'info_table' table - # - # @param tableName string table name - # @param info string information on the table origin - # - def updateInfoTable( self, tableName, info ): - if not self.doesTableExist( "info_tables" ): - sqlCmd = "CREATE TABLE info_tables ( name varchar(255), file varchar(255) )" - self.execute( sqlCmd ) - sqlCmd = 'INSERT INTO info_tables VALUES ("%s","%s")' % (tableName, info) - self.execute( sqlCmd ) - - - ## Get a list with the fields - # - def getFieldList( self, table ): - lFields = [] - sqlCmd = "DESCRIBE %s" % ( table ) - self.execute( sqlCmd ) - lResults = self.fetchall() - for res in lResults: - lFields.append( res[0] ) - return lFields - - - ## Check that the input file has as many fields than it is supposed to according to its format - # - # @note fields should be separated by tab - # - def checkDataFormatting( self, dataType, fileName ): - dataType = dataType.lower() - if dataType in ["fa", "fasta", "seq", "classif", "length", "jobs", "pathstat"]: - return - dDataType2NbFields = { "map": 4, "set": 5, "align": 9, "path": 10, "match": 15, "tab": 15 } - fileHandler = open( fileName, "r" ) - line = fileHandler.readline() - if line != "": - tokens = line.split("\t") - if len(tokens) < dDataType2NbFields[ dataType ]: - msg = "ERROR: '%s' file has less than %i fields" % ( dataType, dDataType2NbFields[ dataType ] ) - self._log.error(msg) - raise RepetException(msg) - if len(tokens) > dDataType2NbFields[ dataType ]: - msg = "ERROR: '%s' file has more than %i fields" % ( dataType, dDataType2NbFields[ dataType ] ) - self._log.error(msg) - raise RepetException(msg) - fileHandler.close() - - - def createIndex(self, tableName="", tableType=""): - sqlCmd = "SHOW INDEX FROM %s;"% (tableName) - self.execute(sqlCmd) - res = self.fetchall() - lIndex = [] - for i in res: - lIndex.append(i[2]) - self._log.warning("existing indexes:", lIndex) - - for indexName, fieldNames in TABLE_INDEX_DESCRIPTOR.get(tableType): - if not indexName in lIndex: - sqlCmd = "CREATE INDEX %s ON %s ( %s );" % (indexName, tableName, fieldNames) - self.execute(sqlCmd) - - - ## Create a MySQL table of specified data type and load data - # - # @param tableName string name of the table to be created - # @param fileName string name of the file containing the data to be loaded in the table - # @param dataType string type of the data (map, set, align, path, match, seq, length, jobs) - # @param overwrite boolean (default = False) - # - def createTable(self, tableName, dataType, fileName = "", overwrite = False): - self._log.info("creating table '%s' from file '%s' of type '%s'..." % (tableName, fileName, dataType)) - - if fileName != "": - self.checkDataFormatting(dataType, fileName) - - if overwrite: - self.dropTable(tableName) - - tableType = dataType.lower() - if TABLE_SCHEMA_DESCRIPTOR.get(tableType,None) is None and TABLE_TYPE_SYNONYMS.get(tableType,None) is None: - msg = "ERROR: unknown type %s" % dataType - self._log.error(msg) - raise RepetException(msg) - - tableType = TABLE_TYPE_SYNONYMS.get(tableType,tableType) - - fields = [" ".join(fieldDescription) for fieldDescription in TABLE_SCHEMA_DESCRIPTOR.get(tableType)] - sqlCmd = "CREATE TABLE %s (%s)" % (tableName, ",".join(fields)) - self.execute(sqlCmd) - self.createIndex(tableName, tableType) - - tmpFileName = "" - if fileName: - if tableType == "seq": - tmpFileName = "%s.seq" % os.path.basename(fileName) - self._convertFastaToSeq(fileName, tmpFileName) - fileName = tmpFileName - elif tableType == "length": - tmpFileName = "%s.length" % os.path.basename(fileName) - self._convertFastaToLength(fileName, tmpFileName) - fileName = tmpFileName - - hasHeaderLine = tableType == "match" or tableType == "pathstat" - self.loadDataFromFile(tableName, fileName, hasHeaderLine) - if tmpFileName: - os.remove(tmpFileName) - - if tableType == "path": - self.changePathQueryCoordinatesToDirectStrand( tableName ) - - self.updateInfoTable(tableName, fileName) - self._log.info("creating table '%s' done!" % tableName) - - - ## Create a bin table for fast access - # - # @param pathTableName string path table name (input table) - # @param idxTableName string bin path table name (output table) - # @param overwrite boolean default = False - # - def createBinPathTable(self, pathTableName, overwrite = False): - idxTableName = "%s_idx" % pathTableName # is an attribute in TableBinPathAdaptator - if not self.doesTableExist(pathTableName): - msg = "ERROR: '%s' doesn't exist => '%s' can't be created" % (pathTableName, idxTableName) - self._log.error(msg) - raise RepetException(msg) - self._log.info("creating %s for fast access" % idxTableName) - if overwrite: - self.dropTable(idxTableName) - - sqlCmd = "CREATE TABLE %s ( path int unsigned, idx int unsigned, contig varchar(255), min int, max int, strand int unsigned)" % idxTableName - self.execute(sqlCmd) - - sqlCmd = "CREATE INDEX id ON %s ( path );" % idxTableName - self.execute(sqlCmd) - sqlCmd = "CREATE INDEX ibin ON %s ( idx );" % idxTableName - self.execute(sqlCmd) - sqlCmd = "CREATE INDEX icontig ON %s ( contig );" % idxTableName - self.execute(sqlCmd) - sqlCmd = "CREATE INDEX imin ON %s ( min );" % idxTableName - self.execute(sqlCmd) - sqlCmd = "CREATE INDEX imax ON %s ( max );" % idxTableName - self.execute(sqlCmd) - sqlCmd = "CREATE INDEX istrand ON %s ( strand );" % idxTableName - self.execute(sqlCmd) - - tmpTableName = "%s_tmp" % pathTableName - self._createPathTableAndGroupByIdAndOrderByStrand(pathTableName, tmpTableName) - iTPA = TablePathAdaptator(self, tmpTableName) - if not self.isEmpty(tmpTableName): - tmpFileName = "%s.tmp%s" % (pathTableName, str(os.getpid())) - with open(tmpFileName, "w") as f: - lQueryNames = iTPA.getQueryList() - for queryName in lQueryNames: - lPaths = iTPA.getPathListFromQuery(queryName) - for i in lPaths: - idx = i.range_query.findIdx() - max = i.range_query.getMax() - min = i.range_query.getMin() - strand = i.range_query.isOnDirectStrand() - f.write("%d\t%d\t%s\t%d\t%d\t%d\n"%(i.id, idx, i.range_query.seqname, min, max, strand)) - sqlCmd="LOAD DATA LOCAL INFILE '%s' INTO TABLE %s FIELDS ESCAPED BY '' " % (tmpFileName, idxTableName) - self.execute(sqlCmd) - self.updateInfoTable(idxTableName, "%s bin indexes" % pathTableName) - os.remove(tmpFileName) - self.dropTable(tmpTableName) - - - ## This table summarize the Path list information according to the identifier numbers. The min and max value is taken - # - def _createPathTableAndGroupByIdAndOrderByStrand(self, pathTableName, outTableName): - self.dropTable(outTableName) - - sqlcmd="CREATE TABLE %s SELECT path, query_name, min(query_start) AS query_start, max(query_end) AS query_end, subject_name, min(subject_start) AS subject_start, max(subject_end) AS subject_end, min(e_value) AS e_value, sum(score) AS score, avg(identity) AS identity FROM %s WHERE query_start<query_end and subject_start<subject_end group by path;" % (outTableName, pathTableName) - self.execute(sqlcmd) - - sqlcmd="INSERT INTO %s SELECT path, query_name, min(query_start) AS query_start, max(query_end) AS query_end, subject_name, max(subject_start) AS subject_start, min(subject_end) AS subject_end, min(e_value) AS e_value, sum(score) AS score, avg(identity) AS identity FROM %s WHERE query_start<query_end and subject_start>subject_end group by path;" % (outTableName, pathTableName) - self.execute(sqlcmd) - - sqlcmd="INSERT INTO %s SELECT path, query_name, max(query_start) AS query_start, min(query_end) AS query_end, subject_name, min(subject_start) AS subject_start, max(subject_end) AS subject_end, min(e_value) AS e_value, sum(score) AS score, avg(identity) AS identity FROM %s WHERE query_start>query_end and subject_start<subject_end group by path;" % (outTableName, pathTableName) - self.execute(sqlcmd) - - sqlcmd="INSERT INTO %s SELECT path, query_name, max(query_start) AS query_start, min(query_end) AS query_end, subject_name, max(subject_start) AS subject_start, min(subject_end) AS subject_end, min(e_value) AS e_value, sum(score) AS score, avg(identity) AS identity FROM %s WHERE query_start>query_end and subject_start>subject_end group by path;" % (outTableName, pathTableName) - self.execute(sqlcmd) - - self.createIndex(outTableName, "path") - - - ## Create a bin table for fast access - # - # @param setTableName string set table name (input table) - # @param idxTableName string bin set table name (output table) - # @param overwrite boolean default = False - # - def createBinSetTable(self, setTableName, overwrite = False): - idxTableName = "%s_idx" % setTableName # is an attribute in TableBinSetAdaptator - if not self.doesTableExist(setTableName): - msg = "ERROR: '%s' doesn't exist => '%s' can't be created" % (setTableName, idxTableName) - self._log.error(msg) - raise RepetException(msg) - self._log.info("creating %s for fast access" % idxTableName) - if overwrite: - self.dropTable(idxTableName) - - sqlCmd = "CREATE TABLE %s ( path int unsigned, bin float, contig varchar(255), min int, max int, strand int unsigned)" % idxTableName - self.execute(sqlCmd) - - sqlCmd = "CREATE INDEX id ON %s ( path );" % idxTableName - self.execute(sqlCmd) - sqlCmd = "CREATE INDEX ibin ON %s ( bin );" % idxTableName - self.execute(sqlCmd) - sqlCmd = "CREATE INDEX icontig ON %s ( contig );" % idxTableName - self.execute(sqlCmd) - sqlCmd = "CREATE INDEX imin ON %s ( min );" % idxTableName - self.execute(sqlCmd) - sqlCmd = "CREATE INDEX imax ON %s ( max );" % idxTableName - self.execute(sqlCmd) - sqlCmd = "CREATE INDEX istrand ON %s ( strand );" % idxTableName - self.execute(sqlCmd) - - tmpTableName = "%s_tmp" % setTableName - self._createSetTableAndGroupByIdAndOrderByStrand(setTableName, tmpTableName) - iTSA = TableSetAdaptator(self, tmpTableName) - if not self.isEmpty(tmpTableName): - tmpFileName = "%s.tmp%s" % (setTableName, str(os.getpid())) - with open(tmpFileName, "w") as f: - lSeqNames = iTSA.getSeqNameList() - for seqName in lSeqNames: - lSets = iTSA.getSetListFromSeqName(seqName) - for i in lSets: - bin = i.getBin() - max = i.getMax() - min = i.getMin() - strand = i.isOnDirectStrand() - f.write("%d\t%f\t%s\t%d\t%d\t%d\n"%(i.id, bin, i.seqname, min, max, strand)) - sqlCmd="LOAD DATA LOCAL INFILE '%s' INTO TABLE %s FIELDS ESCAPED BY '' " % (tmpFileName, idxTableName) - self.execute(sqlCmd) - self.updateInfoTable(idxTableName, "%s bin indexes" % setTableName) - os.remove(tmpFileName) - self.dropTable(tmpTableName) - - - ## This table summarize the Set list information according to the identifier numbers. The min and max value is taken - # - def _createSetTableAndGroupByIdAndOrderByStrand(self, setTableName, outTableName): - self.dropTable(outTableName) - - sqlcmd="CREATE TABLE %s SELECT path, name, chr, min(start) AS start, max(end) AS end FROM %s WHERE start<end group by path;" % (outTableName, setTableName) - self.execute(sqlcmd) - - sqlcmd="INSERT INTO %s SELECT path, name, chr, max(start) AS start, min(end) AS end FROM %s WHERE start>end group by path;" % (outTableName, setTableName) - self.execute(sqlcmd) - - self.createIndex(outTableName, "set") - - - ## Load data from a file into a MySQL table - # - # @param tableName string table name - # @param fileName string file name - # @param escapeFirstLine boolean True to ignore the first line of file, False otherwise - # - def loadDataFromFile(self, tableName, fileName, escapeFirstLine = False): - if fileName != "": - sqlCmd = "LOAD DATA LOCAL INFILE '%s' INTO TABLE %s FIELDS ESCAPED BY '' " % ( fileName, tableName ) - if escapeFirstLine == True: - sqlCmd = "%s IGNORE 1 LINES" %(sqlCmd) - self.execute( sqlCmd ) - - self._log.info("%i entries in the table %s" % (self.getSize(tableName), tableName)) - -###################################################################################### -#TODO: remove duplication with same methods in fastautils - ## Convert a fasta file to a length file - # - # @param inFile string name of the input fasta file - # @param outFile string name of the output file - # - def _convertFastaToLength(self, inFile, outFile = ""): - if outFile == "": - outFile = "%s.length" % inFile - - if inFile != "": - with open(inFile, "r") as inFH: - with open(outFile, "w") as outFH: - bioseq = Bioseq() - while True: - bioseq.read(inFH) - if bioseq.sequence == None: - break - seqLen = bioseq.getLength() - outFH.write("%s\t%d\n" % (bioseq.header.split()[0], seqLen)) - - - ## Convert a fasta file to a seq file - # - # @param inFile string name of the input fasta file - # @param outFile string name of the output file - # - def _convertFastaToSeq(self, inFile, outFile = ""): - if outFile == "": - outFile = "%s.seq" % inFile - - if inFile != "": - with open(inFile, "r") as inFH: - with open(outFile, "w") as outFH: - bioseq = Bioseq() - while True: - bioseq.read(inFH) - if bioseq.sequence == None: - break - seqLen = bioseq.getLength() - outFH.write("%s\t%s\t%s\t%d\n" % (bioseq.header.split()[0], \ - bioseq.sequence, bioseq.header, seqLen)) - -###################################################################################### - - ## Change the coordinates such that the query is on the direct strand. - # - # @param inTable string path table name to update - # - def changePathQueryCoordinatesToDirectStrand( self, inTable ): - sqlCmd = "ALTER TABLE %s ADD COLUMN tmpid INT NOT NULL AUTO_INCREMENT PRIMARY KEY" % ( inTable ) - self.execute( sqlCmd ) - - tmpTable = "%s_tmp" % ( inTable ) - sqlCmd = "CREATE TABLE %s SELECT * FROM %s WHERE query_start > query_end" % ( tmpTable, inTable ) - self.execute( sqlCmd ) - - sqlCmd = "UPDATE %s, %s" % ( inTable, tmpTable ) - sqlCmd += " SET %s.query_start=%s.query_end," % ( inTable, tmpTable ) - sqlCmd += " %s.query_end=%s.query_start," % ( inTable, tmpTable ) - sqlCmd += " %s.subject_start=%s.subject_end," % ( inTable, tmpTable ) - sqlCmd += " %s.subject_end=%s.subject_start" % ( inTable, tmpTable ) - sqlCmd += " WHERE %s.tmpid=%s.tmpid" % ( inTable, tmpTable ) - self.execute( sqlCmd ) - - sqlCmd = "ALTER TABLE %s DROP COLUMN tmpid" % ( inTable ) - self.execute( sqlCmd ) - self.dropTable( tmpTable ) - - - ## Export data from a table in a file. - # - # @param tableName string table name - # @param outFileName string output file name - # @param keepFirstLine boolean if you want the first line (column name) in output file - # @param param string sql parameters to select data expected - # - def exportDataToFile( self, tableName, outFileName="", keepFirstLine=False, param="" ): - if outFileName == "": outFileName = tableName - prg = "mysql" - cmd = prg - cmd += " -h %s" % ( self.host ) - cmd += " -u %s" % ( self.user ) - cmd += " -p\"%s\"" % ( self.passwd ) - cmd += " --database=%s" % ( self.dbname ) - cmd += " -e\"SELECT * FROM %s" % ( tableName ) - if param != "": cmd += " %s" % ( param ) - cmd += ";\"" - cmd += " > " - if keepFirstLine == False: - cmd += "%s.tmp" % ( outFileName ) - else: - cmd += "%s" % ( outFileName ) - log = os.system( cmd ) - if log != 0: print "ERROR: mysql returned %i" % ( log ); sys.exit(1) - - if keepFirstLine == False: - tmpFileName = "%s.tmp" % ( outFileName ) - tmpFile = open( tmpFileName, "r" ) - outFile = open( outFileName, "w" ) - i = 0 - for line in tmpFile: - if i > 0: - outFile.write( line ) - i += 1 - tmpFile.close() - outFile.close() - os.remove( tmpFileName ) - - - ## Convert a Path table into an Align table - # - # @param inPathTable string name of the input Path table - # @param outAlignTable string name of the output Align table - # - def convertPathTableIntoAlignTable( self, inPathTable, outAlignTable ): - sqlCmd = "CREATE TABLE %s SELECT query_name,query_start,query_end,subject_name,subject_start,subject_end,E_value,score,identity FROM %s;" % ( outAlignTable, inPathTable ) - self.execute( sqlCmd ) - self.updateInfoTable( outAlignTable, "" ) - - - ## Create a set table from a map table - # - # @param mapTableName string map table name - # @param setTableName string new set table name - # - def convertMapTableIntoSetTable( self, mapTableName, setTableName ): - sqlCmd = "CREATE TABLE %s (path int(10) unsigned auto_increment primary key) select name, chr, start, end from %s;" % (setTableName, mapTableName) - self.execute(sqlCmd) - self.createIndex(setTableName, "set") - - - ## Convert an Align table into a Path table - # - # @param inAlignTable string name of the input Align table - # @param outPathTable string name of the output Path table - # - def convertAlignTableIntoPathTable( self, inAlignTable, outPathTable ): - self.createTable( outPathTable, "path", "", True ) - sqlCmd = "SELECT * FROM %s" % ( inAlignTable ) - self.execute( sqlCmd ) - lResults = self.fetchall() - rowIndex = 0 - for res in lResults: - rowIndex += 1 - sqlCmd = "INSERT INTO %s" % ( outPathTable ) - sqlCmd += " (path,query_name,query_start,query_end,subject_name,subject_start,subject_end,E_value,score,identity)" - sqlCmd += " VALUES ( '%i'" % ( rowIndex ) - for i in res: - sqlCmd += ', "%s"' % ( i ) - sqlCmd += " )" - self.execute( sqlCmd ) - self.updateInfoTable( outPathTable, "" ) - - - ## Give a list of instances according to the SQL command - # - # @param SQLCmd string is a SQL command - # @param methodGetInstance2Adapt a getter method name. With this method you choose the type of intances contained in lObjs. See example in Test_DbMySql.py. - # @return lObjs list of instances - # - def getObjectListWithSQLCmd( self, SQLCmd, methodGetInstance2Adapt): - self.execute( SQLCmd ) - res = self.fetchall() - lObjs = [] - for t in res: - iObj = methodGetInstance2Adapt() - iObj.setFromTuple( t ) - lObjs.append( iObj ) - return lObjs - - - ## Give a list of integer according to the SQL command - # - # @param sqlCmd string is a SQL command - # @return lInteger integer list - # - def getIntegerListWithSQLCmd( self, sqlCmd ): - self.execute(sqlCmd) - res = self.fetchall() - lInteger = [] - for t in res: - if t[0] != None: - lInteger.append(int(t[0])) - return lInteger - - - ## Give a int according to the SQL command - # - # @param sqlCmd string is a SQL command - # @return nb integer - # - def getIntegerWithSQLCmd( self, sqlCmd ): - self.execute(sqlCmd) - res = self.fetchall() - nb = res[0][0] - if nb == None: - nb = 0 - return nb - - - ## Give a list of str according to the SQL command - # - # @param sqlCmd string is a SQL command - # @return lString str list - # - def getStringListWithSQLCmd( self, sqlCmd ): - self.execute(sqlCmd) - res = self.fetchall() - lString = [] - for i in res: - lString.append(i[0]) - return lString - -#TODO: use API to add indexes - ## Remove doublons in a given table - # - # @param table string name of a MySQL table - # - def removeDoublons( self, table ): - tmpTable = "%s_%s" % ( table, time.strftime("%Y%m%d%H%M%S") ) - sqlCmd = "CREATE TABLE %s SELECT DISTINCT * FROM %s" % ( tmpTable, table ) - self.execute( sqlCmd ) - self.dropTable( table ) - self.renameTable(tmpTable, table) - - - ## Get a list of table names from a pattern - # - # @note for instance pattern = 'MyProject_%' - # - def getTableListFromPattern( self, pattern ): - if pattern == "*" or pattern == "%": - sqlCmd = "SHOW TABLES" - else: - sqlCmd = "SHOW TABLES like '%s'" % ( pattern ) - lTables = self.getStringListWithSQLCmd( sqlCmd ) - return lTables
--- a/commons/core/sql/DbSQLite.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,173 +0,0 @@ -import sqlite3 -import os -import sys - -#TODO: update...compare with DbMySql.py -class DbSQLite(object): - - ## Constructor - # - # @param host string db file path - # @param cfgFileName string configuration file name - # - # @note when a parameter is left blank, the constructor is able - # to set attribute values from environment variable: REPET_HOST, - # - def __init__(self, host = ""): - if host != "": - self.host = host - else: - msg = "ERROR: no host specified" - sys.stderr.write( "%s\n" % msg ) - sys.exit(1) - # remove open() and cursor from init() use directly outside this class ... - self.open() - self.cursor = self.db.cursor() - - ## Connect to the DbSQLite database - # - # @param verbose integer (default = 0) - # - def open( self, verbose = 0, nb = 0 ): - try: - #sqlite.connect(":memory:", check_same_thread = False) - self.db = sqlite3.connect(self.host, check_same_thread= False, isolation_level=None, detect_types=sqlite3.PARSE_DECLTYPES) - except sqlite3.Error, e: - if verbose > 0: - print "ERROR %s" % e - sys.stdout.flush() - return False - return True - - ## Execute a SQL query - # - # @param qry string SQL query to execute - # @param params parameters of SQL query - # - def execute( self, qry, params=None ): - try : - if params == None: - self.cursor.execute( qry ) - else: - self.cursor.execute( qry, params ) - except Exception, e: - #TODO Must be test - try : - if params == None: - self.cursor.execute( qry ) - else: - self.cursor.execute( qry, params ) - except Exception, e: - print "Erreur : %s" % e - - ## Retrieve the results of a SQL query - # - def fetchall(self): - return self.cursor.fetchall() - - ## Record a new table in the 'info_table' table - # - # @param tableName string table name - # @param info string information on the table origin - # - def updateInfoTable( self, tableName, info ): - if not self.doesTableExist( "info_tables" ): - sqlCmd = "CREATE TABLE info_tables ( name varchar(255), file varchar(255) )" - self.execute( sqlCmd ) - sqlCmd = 'INSERT INTO info_tables VALUES ("%s","%s")' % (tableName, info) - self.execute( sqlCmd ) - - def createTable(self, tableName, dataType, overwrite=False, verbose=0): - if verbose > 0: - print "creating table '%s' from file '%s' of type '%s'..." % (tableName, dataType) - sys.stdout.flush() - if overwrite: - self.dropTable(tableName) - if dataType.lower() in ["job", "jobs"]: - self.createJobTable(tableName) - else: - print "ERROR: unknown type %s" % (dataType) - self.close() - sys.exit(1) - if verbose > 0: - print "done!"; sys.stdout.flush() - - ## Create a job table - # - # @param tablename new table name - # - def createJobTable( self, tablename ): - sqlCmd = "CREATE TABLE %s" % ( tablename ) - sqlCmd += " ( jobid INT UNSIGNED" - sqlCmd += ", jobname VARCHAR(255)" - sqlCmd += ", groupid VARCHAR(255)" - sqlCmd += ", command TEXT" - sqlCmd += ", launcher VARCHAR(1024)" - sqlCmd += ", queue VARCHAR(255)" - sqlCmd += ", status VARCHAR(255)" - sqlCmd += ", time timestamp" - sqlCmd += ", node VARCHAR(255) )" - self.execute( sqlCmd ) - - self.updateInfoTable( tablename, "job table" ) - sqlCmd = "CREATE INDEX igroupid ON " + tablename + " ( groupid )" - self.execute( sqlCmd ) - - ## Test if a table exists - # - # @param table string table name - # @return boolean True if the table exists, False otherwise - # - def doesTableExist( self, table ): - qry = "PRAGMA table_info(%s)" % (table) - self.execute( qry ) - results = self.cursor.fetchall() - if results: - return True - return False - - def isEmpty( self, tableName ): - return self.getSize( tableName ) == 0 - - ## Give the rows number of the table - # - # @param tableName string table name - # - def getSize( self, tableName ): - qry = "SELECT count(*) FROM %s;" % ( tableName ) - self.execute( qry ) - res = self.fetchall() - return int( res[0][0] ) - - ## Remove a table if it exists - # - # @param table string table name - # @param verbose integer (default = 0) - # - def dropTable( self, table, verbose = 0 ): - if self.doesTableExist( table ): - sqlCmd = "DROP TABLE %s" % ( table ) - self.execute( sqlCmd ) - sqlCmd = 'DELETE FROM info_tables WHERE name = "%s"' % ( table ) - self.execute( sqlCmd ) - - ## Get a list with the fields - # - def getFieldList( self, table ): - lFields = [] - sqlCmd = "PRAGMA table_info(%s)" % ( table ) - self.execute( sqlCmd ) - lResults = self.fetchall() - for res in lResults: - lFields.append( res[1] ) - return lFields - - ## delete this SQLite database session - # - def delete(self): - os.remove(self.host) - - ## Close the connection - # - def close( self ): - self.db.close()
--- a/commons/core/sql/ITableMapAdaptator.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,113 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - - -## Interface for TableMapAdaptator -# -class ITableMapAdaptator(object): - - ## Insert a map instance - # - # @param obj map or set - # @param delayed boolean must the insert be delayed - # - # @warning old name was insAMap - # - def insert(self, obj, delayed=False): - pass - - - ## Insert a list of Map or Set or Match instances - # - # @param l a list of object instances - # @param delayed boolean - # - # @warning old name was insMapList - # - def insertList(self, l, delayed = False): - pass - - ## Give a list of the distinct seqName/chr present in the table - # - # @return lDistinctContigNames string list - # - # @warning old name was getContig_name - # - def getSeqNameList(self): - pass - - - ## Give a list of Map instances having a given seq name - # - # @param seqName string seq name - # @return lMap list of instances - # - # @warning old name was get_MapList_from_contig - # - def getMapListFromSeqName(self, seqName): - pass - - - ## Return a list of Set instances from a given sequence name - # - # @param seqName string sequence name - # @return lSets list of Set instances - # - # @warning old name was getSetList_from_contig - # - def getSetListFromSeqName( self, seqName ): - pass - - - ## Give a map instances list overlapping a given region - # - # @param seqName string seq name - # @param start integer start coordinate - # @param end integer end coordinate - # @return lMap list of map instances - # - # @warning old name was getMapList_from_qcoord - # - def getMapListOverlappingCoord(self, seqName, start, end): - pass - - - ## Return a list of Set instances overlapping a given region - # - # @param seqName string sequence name - # @param start integer start coordinate - # @param end integer end coordinate - # @return lSet list of Set instances - # - # @warning old name was getSetList_from_qcoord - # - def getSetListOverlappingCoord( self, seqName, start, end ): - pass - \ No newline at end of file
--- a/commons/core/sql/ITableMatchAdaptator.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,68 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - - -## Interface for TableMatchAdaptator -# -class ITableMatchAdaptator(object): - - ## Give a list of Match instances given a query name - # - # @param query string sequence name - # @return lMatches list of Match instances - # - def getMatchListFromQuery( self, query ): - pass - - ## Give a list of Match instances having the same identifier - # - # @param id integer identifier number - # @return lMatch a list of Match instances - # - def getMatchListFromId( self, id ): - pass - - ## Insert a Match instance - # - # @param iMatch a Match instance - # @param delayed boolean - # - def insert(self, iMatch, delayed = False): - pass - - ## Insert a list of Map or Set or Match instances - # - # @param l a list of object instances - # @param delayed boolean - # - # @warning old name was insMapList - # - def insertList(self, l, delayed = False): - pass \ No newline at end of file
--- a/commons/core/sql/ITablePathAdaptator.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,429 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - - -## Interface for TablePathAdaptator -# -class ITablePathAdaptator (object): - - ## Give the data contained in the table as a list of Path instances - # - # @return lPaths list of path instances - # - def getListOfAllPaths( self ): - pass - - ## Give a list of Path instances having the same identifier - # - # @param id integer identifier number - # @return lPath a list of Path instances - # - # @warning old name was getPathList_from_num - # - def getPathListFromId( self, id ): - pass - - ## Give a list of Path instances according to the given list of identifier numbers - # - # @param lId integer list - # @return lPath a list of Path instances - # - # @warning old name was getPathList_from_numlist - # - def getPathListFromIdList( self, lId ): - pass - - ## Give a list of Path instances having the same given query name - # - # @param query string name of the query - # @return lPath a list of Path instances - # - # @warning old name was getPathList_from_query - # - def getPathListFromQuery( self, query ): - pass - - ## Give a list with all the distinct identifiers corresponding to the query - # - # @param query string name of the query - # @return lId a list of integer - # - # @warning old name was getPathList_from_query - # - def getIdListFromQuery( self, query ): - pass - - ## Give a list with all the distinct identifiers corresponding to the subject - # - # @param subject string name of the subject - # @return lId a list of integer - # - # @warning old name was getPathList_from_subject - # - def getIdListFromSubject( self, subject ): - pass - - ## Insert a path instance - # - # @param obj a path instance - # @param delayed boolean indicating if the insert must be delayed - # - # @note data are inserted such that the query is always on the direct strand - # - # @warning old name was insAPath - # - def insert(self, obj, delayed = False): - pass - - ## Insert a list of Path instances - # - # @param l a list of Path instances - # @param delayed boolean - # - # @warning old name was insPathList - # - def insertList(self, l, delayed = False): - pass - - ## Give a list of the identifier number contained in the table - # - # @return l integer list - # - # @warning old name was getPath_num - # - def getIdList(self): - pass - - ## Give a list of Path instances having the same given subject name - # - # @param subject string name of the subject - # @return lPath a list of Path instances - # - # @warning old name was getPath_num - # - def getPathListFromSubject( self, subject ): - pass - - ## Give a list of the distinct subject names present in the table - # - # @return lDistinctTypeNames string list - # - # @warning old name was getListDistinctSubjectName - # - def getSubjectList(self): - pass - - ## Give a list of the distinct query names present in the table - # - # @return lDistinctQueryNames string list - # - # @warning old name was getListDistinctQueryName - # - def getQueryList(self): - pass - - ## Give a list of Set instance list from the path contained on a query name - # - # @param queryName string query name - # @return lSet list of set instance - # - def getSubjectListFromQuery (self, queryName): - pass - - ## Give a list of Path instances with the given query and subject, both on direct strand - # - # @param query string query name - # @param subject string subject name - # @return lPaths list of path instances - # - # @warning old name was getListPathsWithDirectQueryDirectSubjectPerQuerySubject - # - def getPathListWithDirectQueryDirectSubjectFromQuerySubject( self, query, subject ): - pass - - ## Give a list of Path instances with the given query on direct strand and the given subject on reverse strand - # - # @param query string query name - # @param subject string subject name - # @return lPaths list of path instances - # - # @warning old name was getListPathsWithDirectQueryReverseSubjectPerQuerySubject - # - def getPathListWithDirectQueryReverseSubjectFromQuerySubject( self, query, subject ): - pass - - ## Give the number of Path instances with the given query name - # - # @param query string query name - # @return pathNb integer the number of Path instances - # - # @warning old name was getNbPaths_from_query - # - def getNbPathsFromQuery( self, query ): - pass - - ## Give the number of Path instances with the given subject name - # - # @param subject string subject name - # @return pathNb integer the number of Path instances - # - # @warning old name was getNbPaths_from_subject - # - def getNbPathsFromSubject( self, subject ): - pass - - ## Give the number of distinct path identifiers - # - # @return idNb integer the number of Path instances - # - # @warning old name was getNbAllPathsnums - # - def getNbIds( self ): - pass - - ## Give the number of distinct path identifiers for a given subject - # - # subjectName string subject name - # @return idNb integer the number of Path instances - # - # @warning old name was getNbPathsnums_from_subject - # - def getNbIdsFromSubject( self, subjectName ): - pass - - ## Give the number of distinct path identifiers for a given query - # - # @param queryName string query name - # @return idNb integer the number of Path instances - # - # @warning old name was getNbPathsnums_from_query - # - def getNbIdsFromQuery( self, queryName ): - pass - - ## Give a list of Path instances overlapping a given region - # - # @param query string query name - # @param start integer start coordinate - # @param end integer end coordinate - # @return lPath list of Path instances - # - def getPathListOverlappingQueryCoord( self, query, start, end ): - pass - - ## Give a list of Set instances overlapping a given region - # - # @param query string query name - # @param start integer start coordinate - # @param end integer end coordinate - # @return lSet list of Set instances - # - # @warning old name was getSetList_from_qcoord - # - def getSetListOverlappingQueryCoord(self, query, start, end): - pass - - ## Give a list of Path instances included in a given query region - # - # @param query string query name - # @param start integer start coordinate - # @param end integer end coordinate - # @return lPaths list of Path instances - # - # @warning old name was getIncludedPathList_from_qcoord - # - def getPathListIncludedInQueryCoord( self, query, start, end ): - pass - - ## Give a list of Set instances included in a given region - # - # @param query string query name - # @param start integer start coordinate - # @param end integer end coordinate - # @return lSet list of Set instances - # - # @warning old name was getInSetList_from_qcoord - # - def getSetListIncludedInQueryCoord(self, query, start, end): - pass - - ## Give a a list of Path instances sorted by query coordinates - # - # @return lPaths list of Path instances - # - # @warning old name was getListOfPathsSortedByQueryCoord - # - def getPathListSortedByQueryCoord( self ): - pass - - ## Give a a list of Path instances sorted by query coordinates for a given query - # - # @param queryName string query name - # @return lPaths list of Path instances - # - def getPathListSortedByQueryCoordFromQuery( self, queryName ): - pass - - ## Give a list of path instances sorted by increasing E-value - # - # queryName string query name - # @return lPaths list of path instances - # - def getPathListSortedByIncreasingEvalueFromQuery( self, queryName ): - pass - - ## Give a cumulative length of all paths (fragments) for a given subject name - # - # @param subjectName string subject name - # @return nb Cumulative length for all path - # @warning doesn't take into account the overlaps !! - # @warning old name was getCumulPathLength_from_subject - # - def getCumulLengthFromSubject( self, subjectName ): - pass - - ## Give a list of the length of all chains of paths for a given subject name - # - # @param subjectName string name of the subject - # @return lChainLengths list of lengths per chain of paths - # @warning doesn't take into account the overlaps !! - # @warning old name was getListChainLength_from_subject - # - def getChainLengthListFromSubject( self, subjectName ): - pass - - ## Give a list of identity of all chains of paths for a given subject name - # - # @param subjectName string name of the subject - # @return lChainIdentities list of identities per chain of paths - # @warning doesn't take into account the overlaps !! - # @warning old name was getListChainIdentity_from_subject - # - def getChainIdentityListFromSubject( self, subjectName ): - pass - - ## Give a list of Path lists sorted by weighted identity. - # - # @param qry query name - # @return lChains list of chains - # - def getListOfChainsSortedByAscIdentityFromQuery( self, qry ): - pass - - ## Give a list of the length of all paths for a given subject name - # - # @param subjectName string name of the subject - # @return lPathLengths list of lengths per path - # @warning doesn't take into account the overlaps !! - # @warning old name was getListPathLength_from_subject - # - def getPathLengthListFromSubject( self, subjectName ): - pass - - ## Give a a list with all distinct identifiers for a given subject sorted in decreasing order according to the length of the chains - # - # @return lPathNums a list of paths Id - # - # @warning old name was getPathNumListSortedByDecreasingChainLengthFromSubject - # - def getIdListSortedByDecreasingChainLengthFromSubject( self, subjectName ): - pass - - ## Give a list of Set instance list from the path contained on a query name - # - # @param query string query name - # @return lSet list of set instance - # - # @warning old name was getSetList_from_contig - # - def getSetListFromQuery(self, query): - pass - - ## Delete path corresponding to a given identifier number - # - # @param id integer identifier number - # - # @warning old name was delPath_from_num - # - def deleteFromId(self,id): - pass - - ## Delete path corresponding to a given list of identifier number - # - # @param lId list of identifier number - # - # @warning old name was delPath_from_numlist - # - def deleteFromIdList(self,lId): - pass - - ## Join two path by changing id number of id1 and id2 path to the least of id1 and id2 - # - # @param id1 integer path number - # @param id2 integer path number - # @return newId integer id used to join - # - # @warning old name was joinPath - # - def joinTwoPaths(self,id1,id2): - pass - - ## Get a new id number - # - # @return newId integer new id - # - def getNewId(self): - pass - - ## Test if table is empty - # - def isEmpty( self ): - pass - - ## Create a 'pathRange' table from a 'path' table. - # The output table summarizes the information per identifier. - # The min and max value are taken. - # The identity is averaged over the fragments. - # It may overwrite an existing table. - # - # @param outTable string name of the output table - # @return outTable string Table which summarizes the information per identifier - # - def path2PathRange( self, outTable="" ): - pass - - ## Return the number of times a given instance is present in the table - # The identifier is not considered, - # only coordinates, score, E-value and identity. - # - # @return nbOcc integer - # - def getNbOccurrences( self, iPath ): - pass
--- a/commons/core/sql/ITableSeqAdaptator.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,63 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - - -## Interface for TableSeqAdaptator -# -class ITableSeqAdaptator(object): - - ## Retrieve all the distinct accession names in a list. - # - # @return lAccessions list of accessions - # - # @warning old name was getListAccession - # - def getAccessionsList( self ): - pass - - ## Save sequences in a fasta file from a list of accession names. - # - # @param lAccessions list of accessions - # @param outFileName string Fasta file - # - # @warning old name saveListAccessionInFastaFile - # - def saveAccessionsListInFastaFile( self, lAccessions, outFileName ): - pass - - ## insert bioseq instance - # - # @param seq bioseq - # @param delayed boolean must the insert be delayed - # - # @warning old name was insASeq - # - def insert(self, seq, delayed = False): - pass \ No newline at end of file
--- a/commons/core/sql/ITableSetAdaptator.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,146 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - -## Interface for TableSetAdaptator -# -class ITableSetAdaptator (object): - - ## Insert a set instance - # - # @param obj a set instance - # @param delayed boolean indicating if the insert must be delayed - # - # @warning old name was insASet - # - def insert(self, obj, delayed = False): - pass - - ## Insert a list of Set instances - # - # @param l a list of object instances - # @param delayed boolean - # - # @warning old name was insSetList - # - def insertList(self, l, delayed = False): - pass - - ## Give a list of identifier numbers contained in the table - # - # @return l integer list - # - # @warning old name was getSet_num - # - def getIdList(self): - pass - - ## Give a list of Set instances having a given seq name - # - # @param seqName string seq name - # @return lSets list of instances - # - # @warning old name was get_SetList_from_contig - # - def getSetListFromSeqName(self, seqName): - pass - - ## Give a set instances list with a given identifier number - # - # @param id integer identifier number - # @return lSet list of set instances - # - # @warning old name was getSetList_from_num - # - def getSetListFromId(self, id): - pass - - ## Give a set instances list with a list of identifier numbers - # - # @param lId integers list identifiers list numbers - # @return lSet list of set instances - # - # @warning old name was getSetList_from_numlist - # - def getSetListFromIdList(self,lId): - pass - - ## Return a list of Set instances overlapping a given sequence - # - # @param seqName string sequence name - # @param start integer start coordinate - # @param end integer end coordinate - # @return lSet list of Set instances - # - # @warning old name was getSetList_from_qcoord - # - def getSetListOverlappingCoord( self, seqName, start, end ): - pass - - ## Delete set corresponding to a given identifier number - # - # @param id integer identifier number - # - # @warning old name was delSet_from_num - # - def deleteFromId(self, id): - pass - - ## Delete set corresponding to a given list of identifier number - # - # @param lId integers list list of identifier number - # - # @warning old name was delSet_from_listnum - # - def deleteFromIdList(self, lId): - pass - - ## Join two set by changing id number of id1 and id2 set to the least of id1 and id2 - # - # @param id1 integer id path number - # @param id2 integer id path number - # - # @warning old name was joinSet - # - def joinTwoSets(self, id1, id2): - pass - - ## Get a new id number - # - # @return new_id integer max_id + 1 - # - def getNewId(self): - pass - - ## Give the data contained in the table as a list of Sets instances - # - # @return lSets list of set instances - # - def getListOfAllSets( self ): - pass \ No newline at end of file
--- a/commons/core/sql/Job.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,74 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - -## Job informations to launch a command on a cluster. -# -class Job(object): - - ## Constructor - # - # @param jobid the job identifier - # @param jobname the job name - # @param groupid the group identifier to record related job series - # @param queue queue name of the job manager - # @param command command launched - # @param node cluster node name where the execution takes place - # @param launcherFile file name launched as job - # @param lResources resources (memory, time...) but need to conform to SGE/Torque syntax ! - # - def __init__(self, jobid=0, jobname="", groupid="", queue="", command="", launcherFile="",\ - node="", lResources=["mem_free=1G"], parallelEnvironment="" ): - if str(jobid).isdigit(): - self.jobid = int(jobid) - self.jobname = jobname - else: - self.jobname = jobid - self.jobid = 0 - self.jobid = jobid - self.groupid = groupid - self.setQueue(queue) - self.command = command - self.launcher = launcherFile - self.node = node - self.lResources = lResources - self.parallelEnvironment = parallelEnvironment - - def setQueue(self, queue): - self.queue = "" - if queue != "none": - self.queue = queue - - def __eq__(self, o): - if self.jobid == o.jobid and self.jobname == o.jobname\ - and self.groupid == o.groupid and self.queue == o.queue and self.command == o.command \ - and self.launcher == o.launcher and self.node == o.node and self.lResources == o.lResources \ - and self.parallelEnvironment == o.parallelEnvironment: - return True - return False
--- a/commons/core/sql/JobAdaptator.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,271 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - - -import os -import time -import sys -import tempfile -import subprocess -from commons.core.sql.Job import Job - -## Methods for Job persistence -# -class JobAdaptator(object): - - def __init__(self, lJob = [], table = "" ): - self._lJobID = lJob - self._table = table - self._acronym = "" - ## Record a job - # - # @param job Job instance with the job informations - # - def recordJob(self, job): - self._lJobID.append(job) - - ## Remove a job from the job table - # - # @param job: job instance to remove - # - def removeJob(self, job): - pass - - ## Set the jobid of a job with the id of SGE - # - # @param job job instance - # @param jobid integer - # - def updateJobIdInDB(self, job, jobid): - pass - - ## Get a job status - # - # @param job: a Job instance with the job informations - # - def getJobStatus(self, job): - pass - - - ## Change a job status - # - # @param job: a Job instance with the job informations - # @param status: the new status (waiting,finished,error) - # - def changeJobStatus(self, job, status): - pass - - ## Get the number of jobs belonging to the desired groupid with the desired status. - # - # @param groupid string a group identifier to record related job series - # @param status string job status (waiting, running, finished, error) - # @return int - # - def getCountStatus(self, groupid, status): - pass - - ## Clean all job from a job group - # - # @param groupid: a group identifier to record related job series - # - def cleanJobGroup(self, groupid): - pass - - ## Check if there is unfinished job from a job group. - # - # @param groupid string a group identifier to record related job series - # - def hasUnfinishedJob(self, groupid): - pass - - def _getJobIDListFromQstat(self): - lJobIDFromQstat = [] - tmp = tempfile.NamedTemporaryFile(delete=False) - cmd ="qstat | grep %s" % self._acronym - process = subprocess.Popen(cmd, shell=True,stdout=tmp) - process.communicate() - tmp.close() - if process.returncode == 0: - fileName = tmp.name - jobidFileHandler = open(fileName, "r") - for line in jobidFileHandler: - line2 = line.lstrip(" ") - lJobIDFromQstat.append(line2.split(" ")[0]) - jobidFileHandler.close() - os.remove(fileName) - return lJobIDFromQstat - - def _areJobsStillRunning(self,lJobID,lJobIDFromQstat): - sorted(lJobID) - sorted(lJobIDFromQstat) - for i in lJobID: - for j in lJobIDFromQstat: - if int(i)== int(j): - return True - return False - - - ## Wait job finished status from a job group. - # Job are re-launched if error (max. 3 times) - # - # @param groupid string a group identifier to record related job series - # @param checkInterval integer time laps in seconds between two checks (default = 5) - # @param maxRelaunch integer max nb of times a job in error is relaunch before exiting (default = 3) - # @param exitIfTooManyErrors boolean exit if a job is still in error above maxRelaunch (default = True) - # @param timeOutPerJob integer max nb of seconds after which one tests if a job is still in SGE or not (default = 60*60=1h) - # - def waitJobGroup(self, groupid, checkInterval=5, maxRelaunch=3, exitIfTooManyErrors=True, timeOutPerJob=60*60): - - while True: - time.sleep(checkInterval) - lJobIDFromQstat = self._getJobIDListFromQstat() - if self._areJobsStillRunning(self._lJobID, lJobIDFromQstat) == False: - break - - ## Submit a job to a queue and record it in job table. - # - # @param job a job instance - # @param maxNbWaitingJobs integer max nb of waiting jobs before submitting a new one (default = 10000) - # @param checkInterval integer time laps in seconds between two checks (default = 30) - # @param verbose integer (default = 0) - # - def submitJob(self, job, verbose=0, maxNbWaitingJobs=10000, checkInterval=30): - cmd = self._getQsubCommand(job) - tmp = tempfile.NamedTemporaryFile(delete=False) - process = subprocess.Popen(cmd, shell=True,stdout=tmp) - process.communicate() - tmp.close() - if process.returncode == 0: - fileName = tmp.name - jobidFileHandler = open(fileName, "r") - jobid = self._getJobidFromJobManager(jobidFileHandler) - if verbose > 0: - print "job '%i %s' submitted" % (jobid, job.jobname) - sys.stdout.flush() - job.jobid = jobid - #newJob= Job(job.jobid, job.jobname, job.groupid, job.queue, job.command, job.launcher, job.node, job.lResources, job.parallelEnvironment) - self._acronym = job.jobname.split("_")[0][:10] - self.recordJob(job.jobid) - jobidFileHandler.close() - os.remove(fileName) - return process.returncode - - - ## Get the list of nodes where jobs of one group were executed - # - # @param groupid string a group identifier of job series - # @return lNodes list of nodes names without redundancy - # - def getNodesListByGroupId(self, groupId): - pass - - def checkJobTable(self): - pass - - def close(self): - pass - - def _getJobidAndNbJob(self, jobid) : - tab = jobid.split(".") - jobid = tab[0] - tab = tab[1].split(":") - nbJob = tab[0] - return jobid, nbJob - -class JobAdaptatorSGE(JobAdaptator): - - ## Check if a job is still handled by SGE - # - # @param jobid string job identifier - # @param jobname string job name - # - def isJobStillHandledBySge(self, jobid, jobname): - isJobInQstat = False - tmp = tempfile.NamedTemporaryFile(delete=False) - cmd = "qstat" - process = subprocess.Popen(cmd, shell=True,stdout=tmp) - process.communicate() - tmp.close() - qstatFile = tmp.name - if process.returncode != 0: - msg = "ERROR while launching 'qstat'" - sys.stderr.write( "%s\n" % msg ) - sys.exit(1) - qstatFileHandler = open(qstatFile, "r") - lLines = qstatFileHandler.readlines() - for line in lLines: - tokens = line.split() - if len(tokens) > 3 and tokens[0] == str(jobid) and tokens[2] == jobname[0:len(tokens[2])]: - isJobInQstat = True - break - qstatFileHandler.close() - os.remove(qstatFile) - return isJobInQstat - - def _getQsubCommand(self, job): - cmd = "echo '%s' | " % job.launcher - prg = "qsub" - cmd += prg - cmd += " -V" - cmd += " -N %s" % job.jobname - if job.queue != "": - cmd += " -q %s" % job.queue - cmd += " -cwd" - if job.lResources != []: - cmd += " -l \"" - cmd += " ".join(job.lResources) - cmd += "\"" - if job.parallelEnvironment != "": - cmd += " -pe " + job.parallelEnvironment - return cmd - - def _getJobidFromJobManager(self, jobidFileHandler): - return int(jobidFileHandler.readline().split(" ")[2]) - - -class JobAdaptatorTorque(JobAdaptator): - - def _getQsubCommand(self, job): - cmd = "echo '%s' | " % job.launcher - prg = "qsub" - cmd += prg - cmd += " -V" - cmd += " -d %s" % os.getcwd() - cmd += " -N %s" % job.jobname - if job.queue != "": - cmd += " -q %s" % job.queue - if job.lResources != []: - cmd += " -l \"" - cmd += " ".join(job.lResources).replace("mem_free","mem") - cmd += "\"" - return cmd - - def _getJobidFromJobManager(self, jobidFileHandler): - return int(jobidFileHandler.readline().split(".")[0])
--- a/commons/core/sql/OldRepetDB.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,27 +0,0 @@ -import pyRepet.sql.RepetDBMySQL - - -class RepetDB ( pyRepet.sql.RepetDBMySQL.RepetDB ): - - #TODO: try - def execute( self, qry, params=None ): - if params == None: - self.cursor.execute( qry ) - else: - self.cursor.execute( qry, params ) - - - ## Record a new table in the 'info_table' table - # - # @param tablename table name - # @param info information on the origin of the table - # - def updateInfoTable( self, tablename, info ): - self.execute( """SHOW TABLES""" ) - results = self.fetchall() - if ("info_tables",) not in results: - sqlCmd = "CREATE TABLE info_tables ( name varchar(255), file varchar(255) )" - self.execute( sqlCmd ) - qryParams = "INSERT INTO info_tables VALUES (%s, %s)" - params = ( tablename, info ) - self.execute( qryParams,params )
--- a/commons/core/sql/RepetJob.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,252 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - - -import os -import time -import sys -from commons.core.sql.DbMySql import DbMySql -from commons.core.sql.TableJobAdaptatorFactory import TableJobAdaptatorFactory - -#TODO: to remove... => replace all RepetJob() by TableJobAdaptator()... -## Methods for Job persistence -# -class RepetJob( DbMySql ): - - - ## Record a job - # - # @param job Job instance with the job informations - # - def recordJob( self, job ): - self.removeJob( job ) - sqlCmd = "INSERT INTO %s" % ( job.tablename ) - sqlCmd += " VALUES (" - sqlCmd += " \"%s\"," % ( job.jobid ) - sqlCmd += " \"%s\"," % ( job.jobname ) - sqlCmd += " \"%s\"," % ( job.groupid ) - sqlCmd += " \"%s\"," % ( job.command.replace("\"","\'") ) - sqlCmd += " \"%s\"," % ( job.launcher ) - sqlCmd += " \"%s\"," % ( job.queue ) - sqlCmd += " \"waiting\"," - sqlCmd += " \"%s\"," % ( time.strftime( "%Y-%m-%d %H:%M:%S" ) ) - sqlCmd += " \"?\" );" - self.execute( sqlCmd ) - - - ## Remove a job from the job table - # - # @param job: job instance to remove - # - def removeJob( self, job ): - qry = "DELETE FROM %s" % ( job.tablename ) - qry += " WHERE groupid='%s'" % ( job.groupid ) - qry += " AND jobname='%s'" % ( job.jobname ) - qry += " AND queue='%s';" % ( job.queue ) - self.execute( qry ) - - - ## Set the jobid of a job with the id of SGE - # - # @param job job instance - # @param jobid integer - # - def setJobIdFromSge( self, job, jobid ): - qry = "UPDATE %s" % ( job.tablename ) - qry += " SET jobid='%i'" % ( int(jobid) ) - qry += " WHERE jobname='%s'" % ( job.jobname ) - qry += " AND groupid='%s'" % ( job.groupid ) - qry += " AND queue='%s';" % ( job.queue ) - self.execute( qry ) - - - ## Get a job status - # - # @param job: a Job instance with the job informations - # - def getJobStatus( self, job ): - if job.jobid != 0 and job.jobname == "": - job.jobname = job.jobid - job.jobid = 0 - qry = "SELECT status FROM %s" % ( job.tablename ) - qry += " WHERE groupid='%s'" % ( job.groupid ) - qry += " AND jobname='%s'" % ( job.jobname ) - qry += " AND queue='%s';" % ( job.queue ) - self.execute( qry ) - res = self.fetchall() - if len(res) > 1: - msg = "ERROR while getting job status: non-unique jobs" - sys.stderr.write( "%s\n" % msg ) - sys.stderr.flush() - sys.exit(1) - if res == None or len(res) == 0: - return "unknown" - return res[0][0] - - - ## Change a job status - # - # @param job: a Job instance with the job informations - # @param status: the new status (waiting,finished,error) - # @param method: db or file - # - def changeJobStatus( self, job, status, method=""): - sqlCmd = "UPDATE %s" % ( job.tablename ) - sqlCmd += " SET status='%s'" % ( status ) - sqlCmd += ",node='%s'" % ( job.node ) - sqlCmd += " WHERE groupid='%s'" % ( job.groupid ) - sqlCmd += " AND jobname='%s'" % ( job.jobname ) - sqlCmd += " AND queue='%s';" % ( job.queue ) - self.execute( sqlCmd ) - - - ## Get the number of jobs belonging to the desired groupid with the desired status. - # - # @param tablename string table name to record the jobs - # @param groupid string a group identifier to record related job series - # @param status string job status (waiting, running, finished, error) - # @return int - # - def getCountStatus( self, tablename, groupid, status ): - qry = "SELECT count(jobname) FROM %s" % ( tablename ) - qry += " WHERE groupid='%s'" % ( groupid ) - qry += " AND status='%s';" % ( status ) - self.execute( qry ) - res = self.fetchall() - return int( res[0][0] ) - - - ## Clean all job from a job group - # - # @param tablename table name to record the jobs - # @param groupid: a group identifier to record related job series - # - def cleanJobGroup( self, tablename, groupid ): - if self.doesTableExist( tablename ): - qry = "DELETE FROM %s WHERE groupid='%s';" % ( tablename, groupid ) - self.execute( qry ) - - - ## Check if there is unfinished job from a job group. - # - # @param tablename string table name to record the jobs - # @param groupid string a group identifier to record related job series - # - def hasUnfinishedJob( self, tablename, groupid ): - if not self.doesTableExist( tablename ): - return False - qry = "SELECT * FROM %s" % ( tablename ) - qry += " WHERE groupid='%s'" % ( groupid ) - qry += " and status!='finished';" - self.execute( qry ) - res = self.fetchall() - if len(res) == 0: - return False - return True - - - ## Check if a job is still handled by SGE - # - # @param jobid string job identifier - # @param jobname string job name - # - def isJobStillHandledBySge( self, jobid, jobname ): - isJobInQstat = False - qstatFile = "qstat_stdout" - cmd = "qstat > %s" % ( qstatFile ) - returnStatus = os.system( cmd ) - if returnStatus != 0: - msg = "ERROR while launching 'qstat'" - sys.stderr.write( "%s\n" % msg ) - sys.exit(1) - qstatFileHandler = open( qstatFile, "r" ) - lLines = qstatFileHandler.readlines() - for line in lLines: - tokens = line.split() - if len(tokens) > 3 and tokens[0] == str(jobid) and tokens[2] == jobname[0:len(tokens[2])]: - isJobInQstat = True - break - qstatFileHandler.close() - os.remove( qstatFile ) - return isJobInQstat - - - ## Wait job finished status from a job group. - # Job are re-launched if error (max. 3 times) - # - # @param tableName string table name to record the jobs - # @param groupid string a group identifier to record related job series - # @param checkInterval integer time laps in seconds between two checks (default = 5) - # @param maxRelaunch integer max nb of times a job in error is relaunch before exiting (default = 3) - # @param exitIfTooManyErrors boolean exit if a job is still in error above maxRelaunch (default = True) - # @param timeOutPerJob integer max nb of seconds after which one tests if a job is still in SGE or not (default = 60*60=1h) - # - def waitJobGroup(self, tableName, groupid, checkInterval=5, maxRelaunch=3, exitIfTooManyErrors=True, timeOutPerJob=60*60): - iTJA = TableJobAdaptatorFactory.createInstance(self, tableName) - iTJA.waitJobGroup(groupid, checkInterval, maxRelaunch, exitIfTooManyErrors, timeOutPerJob) - - ## Submit a job to a queue and record it in job table. - # - # @param job a job instance - # @param maxNbWaitingJobs integer max nb of waiting jobs before submitting a new one (default = 10000) - # @param checkInterval integer time laps in seconds between two checks (default = 30) - # @param verbose integer (default = 0) - # - def submitJob( self, job, verbose=0, maxNbWaitingJobs=10000, checkInterval=30 ): - iTJA = TableJobAdaptatorFactory.createInstance(self, job.tablename) - return iTJA.submitJob(job, verbose, maxNbWaitingJobs, checkInterval) - - - ## Get the list of nodes where jobs of one group were executed - # - # @param tablename string table name where jobs are recored - # @param groupid string a group identifier of job series - # @return lNodes list of nodes names - # - def getNodesListByGroupId( self, tableName, groupId ): - qry = "SELECT node FROM %s" % tableName - qry += " WHERE groupid='%s'" % groupId - self.execute( qry ) - res = self.fetchall() - lNodes = [] - for resTuple in res: - lNodes.append(resTuple[0]) - return lNodes - - def getDbName(self): - return "DbMySql" - - def _getJobidAndNbJob(self, jobid) : - tab = [] - tab = jobid.split(".") - jobid = tab[0] - tab = tab[1].split(":") - nbJob = tab[0] - return jobid, nbJob
--- a/commons/core/sql/TableAdaptator.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,128 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - - -## Abstract class, Ancestor of Table*Adaptator -# -class TableAdaptator( object ): - - ## Constructor - # - # @param iDb DbMySql instance - # @param table str table name - # - def __init__( self, iDb = None, table = "" ): - self._iDb = iDb - self._table = table - - ## Set connector to database - # - # @param iDb database instance - # - def setDbConnector( self, iDb ): - self._iDb = iDb - - ## Set table - # - # @param table string table name - # - def setTable( self, table ): - self._table = table - - ## Return the table name - # - def getTable( self ): - return self._table - - ## Return the number of rows in the table - # - def getSize( self ): - return self._iDb.getSize( self._table ) - - ## Test if table is empty - # - def isEmpty( self ): - return self._iDb.isEmpty( self._table ) - - ## Insert an instance of Map or Set or Match or Path or Seq instances - # - # @param obj a Map or Set or Match or Path or Seq instance - # @param delayed boolean - # - def insert(self, obj, delayed = False): - if obj.isEmpty(): - return - self._escapeAntislash(obj) - sql_cmd = self._genSqlCmdForInsert(obj, delayed) - self._iDb.execute(sql_cmd) - - ## Insert a list of Map or Set or Match or Path instances - # - # @param l a list of object instances - # @param delayed boolean - # - def insertList(self, l, delayed = False): - for i in l: - self.insert(i, delayed) - - ## Give the data contained in the table as a list of coord object instances - # - # @return lObject list of coord object instances - # - def getListOfAllCoordObject( self ): - sqlCmd = "SELECT * FROM %s" % ( self._table ) - lObjs = self._iDb.getObjectListWithSQLCmd( sqlCmd, self._getInstanceToAdapt ) - return lObjs - - ## Generate sql command for GetListOverlappingCoord method - # - # @param obj Map, Set or Match instance - # @param delayed boolean - # @return sqlCmd string generated sql command - # - def _genSqlCmdForInsert(self, obj, delayed): - sqlCmd = 'INSERT ' - if delayed : - sqlCmd += ' DELAYED ' - type2Insert, attr2Insert = self._getTypeAndAttr2Insert(obj) - sqlCmd += 'INTO %s VALUES (' % (self._table) - sqlCmd += ",".join(type2Insert) - sqlCmd += ")" - sqlCmd = sqlCmd % attr2Insert - return sqlCmd - - def _getTypeAndAttr2Insert(self, obj): - pass - - def _getInstanceToAdapt(self): - pass - - def _escapeAntislash(self, obj): - pass
--- a/commons/core/sql/TableBinPathAdaptator.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,257 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - -from commons.core.coord.Range import getIdx -from commons.core.sql.TablePathAdaptator import TablePathAdaptator -from commons.core.coord.PathUtils import PathUtils - -## Bin Adaptator for a path table. -# -class TableBinPathAdaptator(TablePathAdaptator): - - - ## Constructor - # - # @param db db instance - # @param tableName string table name (default = "") - # - def __init__(self, db, tableName = ""): - TablePathAdaptator.__init__(self, db, tableName) - self._table_idx = "%s_idx" % (self._table) - - ## Insert a path instance - # - # @param path a path instance - # @param delayed boolean indicating if the insert must be delayed (default = false) - # - def insert( self, path, delayed = False ): - TablePathAdaptator.insert(self, path, delayed) - self._escapeAntislash(path) - idx = path.range_query.findIdx() - max = path.range_query.getMax() - min = path.range_query.getMin() - strand = path.range_query.isOnDirectStrand() - if delayed: - sql_cmd = 'INSERT DELAYED INTO %s VALUES (%d,%d,"%s",%d,%d,%d)'\ - % (self._table_idx,\ - path.id,\ - idx,\ - path.range_query.seqname,\ - min,\ - max,\ - strand) - else: - sql_cmd = 'INSERT INTO %s VALUES (%d,%d,"%s",%d,%d,%d)'\ - % (self._table_idx,\ - path.id,\ - idx,\ - path.range_query.seqname,\ - min,\ - max,\ - strand) - - self._iDb.execute(sql_cmd) - - ## Return a path instances list included in a given region using the bin scheme - # - # @param contig string contig name - # @param start integer start coordinate - # @param end integer end coordinate - # @return lOutPath a path instances list - # - def getPathListIncludedInQueryCoord(self, contig, start, end): - min_coord = min(start, end) - max_coord = max(start, end) - lpath = self.getChainListOverlappingQueryCoord(contig, start, end) - lOutPath = [] - for i in lpath: - if i.range_query.getMin() > min_coord and \ - i.range_query.getMax() < max_coord: - lOutPath.append(i) - - return lOutPath - - ## Return a path instances list overlapping (and included) in a given region using the bin scheme - # - # @param contig string contig name - # @param start integer start coordinate - # @param end integer end coordinate - # @return lOutPath a path instances list - # - def getPathListOverlappingQueryCoord(self, contig, start, end): - min_coord = min(start, end) - max_coord = max(start, end) - lpath = self.getChainListOverlappingQueryCoord(contig, start, end) - lOutPath = [] - for i in lpath: - if ((i.range_query.getMin() <= min_coord and i.range_query.getMax() >= min_coord) or \ - (i.range_query.getMin() >= min_coord and i.range_query.getMin() <= max_coord) or \ - (i.range_query.getMin() <= min_coord and i.range_query.getMax() >= max_coord) or \ - (i.range_query.getMin() >= min_coord and i.range_query.getMax() <= max_coord)) and \ - (i.range_query.getSeqname() == contig): - lOutPath.append(i) - - return lOutPath - - ## Return a path instances list chain (by Id and Coord in chr) list overlapping a given region using the bin scheme - # - # @param contig string contig name - # @param start integer start coordinate - # @param end integer end coordinate - # @return lpath a path instances list - # - def getChainListOverlappingQueryCoord(self, contig, start, end): - min_coord = min(start, end) - max_coord = max(start, end) - sql_cmd = 'select distinct path from %s where contig="%s" and ('\ - % (self._table + "_idx", contig) - - for bin_lvl in xrange(6, 2, -1): - if getIdx(start,bin_lvl) == getIdx(end, bin_lvl): - idx = getIdx(start, bin_lvl) - sql_cmd += 'idx=%d' % (idx) - else: - idx1 = getIdx(min_coord, bin_lvl) - idx2 = getIdx(max_coord, bin_lvl) - sql_cmd += 'idx between %d and %d' % (idx1, idx2) - if bin_lvl > 3: - sql_cmd += " or " - - sql_cmd += ") and min<=%d and max>=%d;" % (max_coord, min_coord) - - - self._iDb.execute(sql_cmd) - res = self._iDb.fetchall() - lnum = [] - for i in res: - lnum.append( int(i[0]) ) - lpath = self.getPathListFromIdList(lnum) - return lpath - - ## Delete path corresponding to a given identifier number - # - # @param num integer identifier number - # - def deleteFromId(self, num): - TablePathAdaptator.deleteFromId(self, num) - sqlCmd='delete from %s where path=%d;' % (self._table_idx, num) - self._iDb.execute(sqlCmd) - - ## Delete path corresponding to a given list of identifier number - # - # @param lNum list list of integer identifier number - # - def deleteFromIdList(self, lNum): - if lNum == []: - return - TablePathAdaptator.deleteFromIdList(self, lNum) - sqlCmd = 'delete from %s where path=%d' % (self._table_idx, lNum[0]) - for i in lNum[1:]: - sqlCmd += " or path=%d" % (i) - sqlCmd += ";" - self._iDb.execute(sqlCmd) - - ## Join two path by changing id number of id1 and id2 path to the least of id1 and id2 - # - # @param id1 integer id path number - # @param id2 integer id path number - # @return newId integer minimum of id1 id2 - # @note this method modify the ID even if this one not existing in the path table - # - def joinTwoPaths(self, id1, id2): - TablePathAdaptator.joinTwoPaths(self, id1, id2) - if id1 < id2: - newId = id1 - oldId = id2 - else: - newId = id2 - oldId = id1 - sqlCmd = 'UPDATE %s SET path=%d WHERE path=%d' % (self._table_idx, newId, oldId) - self._iDb.execute(sqlCmd) - return newId - - ## Get a new id number - # - # @return newId integer max Id in path table + 1 - # - def getNewId(self): - sqlCmd = 'select max(path) from %s;' % (self._table_idx) - self._iDb.execute(sqlCmd) - maxId = self._iDb.fetchall()[0][0] - if maxId == None: - maxId = 0 - newId = int(maxId) + 1 - return newId - - ## Give a list of Set instances included in a given region - # - # @param query string query name - # @param start integer start coordinate - # @param end integer end coordinate - # @return lSet list of Set instances - # - def getSetListIncludedInQueryCoord(self, query, start, end): - lPath=self.getPathListIncludedInQueryCoord(query, start, end) - lSet = PathUtils.getSetListFromQueries(lPath) - return lSet - - ## Give a list of Set instances overlapping a given region - # - # @param query string query name - # @param start integer start coordinate - # @param end integer end coordinate - # @return lSet list of Set instances - # - def getSetListOverlappingQueryCoord(self, query, start, end): - lPath = self.getPathListOverlappingQueryCoord(query, start, end) - lSet = PathUtils.getSetListFromQueries(lPath) - return lSet - - ## Give a list of identifiers contained in the table - # - # @return lId integer list - # - def getIdList(self): - sqlCmd = "SELECT DISTINCT path from %s;" % (self._table_idx) - lId = self._iDb.getIntegerListWithSQLCmd( sqlCmd ) - return lId - - ## Give a list of the distinct query names present in the table - # - # @return lDistinctQueryNames string list - # - def getQueryList(self): - lDistinctQueryNames = self._getDistinctTypeNamesList("query") - return lDistinctQueryNames - - def _getDistinctTypeNamesList( self, type ): - sqlCmd = "SELECT DISTINCT contig FROM %s" % ( self._table_idx ) - lDistinctTypeNames = self._iDb.getStringListWithSQLCmd(sqlCmd) - return lDistinctTypeNames \ No newline at end of file
--- a/commons/core/sql/TableBinSetAdaptator.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,265 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - -from commons.core.sql.TableSetAdaptator import TableSetAdaptator -from commons.core.coord.SetUtils import SetUtils - -## Adaptator for Set tables with bin indexes -# -class TableBinSetAdaptator(TableSetAdaptator): - - ## constructor - # - # @param iDb DbMySql instance instance of DbMySql - # @param tableName string table name (default = "") - # - def __init__(self, iDb, tableName = ""): - TableSetAdaptator.__init__(self, iDb, tableName) - self._table_idx = "%s_idx" % (self._table) - - ## Insert a set instance in a set bin table - # - # @param iSet set instance an instance of set object - # @param delayed boolean an insert delayed or not - # - def insASetInSetAndBinTable(self, iSet, delayed = False): - self.insert(iSet, delayed) - iSet.seqname = iSet.seqname.replace("\\", "\\\\") - iSet.name = iSet.name.replace("\\", "\\\\") - bin = iSet.getBin() - max = iSet.getMax() - min = iSet.getMin() - strand = iSet.isOnDirectStrand() - sql_prefix = '' - if delayed: - sql_prefix = 'INSERT DELAYED INTO ' - else: - sql_prefix = 'INSERT INTO ' - sql_cmd = sql_prefix + '%s VALUES (%d,%f,"%s",%d,%d,%d)'\ - %(self._table_idx,\ - iSet.id,\ - bin,\ - iSet.seqname,\ - min,\ - max,\ - strand) - self._iDb.execute(sql_cmd) - - ## Delete set corresponding to a given identifier number in set and bin set table - # @param id integer identifier number - # @note old name was delSet_from_num - # - def deleteFromIdFromSetAndBinTable(self, id): - self.deleteFromId(id) - sql_cmd = 'delete from %s where path=%d' % (self._table_idx, id) - self._iDb.execute(sql_cmd) - - ## Delete path corresponding to a given list of identifier number - # - # @param lId integer list list of identifier number - # @note old name was delSet_from_listnum - # - def deleteFromListIdFromSetAndBinTable(self, lId): - if lId != []: - self.deleteFromIdList(lId) - sql_cmd = 'delete from %s where path=%d' % (self._table_idx, lId[0]) - for i in lId[1:]: - sql_cmd += " or path=%d" % (i) - self._iDb.execute(sql_cmd) - - ## Join two set by changing id number of id1 and id2 path - # to the least of id1 and id2 - # - # @param id1 integer id path number - # @param id2 integer id path number - # @return id integer new id - # @note old name was joinSet - # - def joinTwoSetsFromSetAndBinTable(self, id1, id2): - self.joinTwoSets(id1, id2) - if id1 < id2: - new_id = id1 - old_id = id2 - else: - new_id = id2 - old_id = id1 - sql_cmd = 'UPDATE %s SET path=%d WHERE path=%d'\ - % (self._table_idx, new_id, old_id) - self._iDb.execute(sql_cmd) - return new_id - - ## Get a new id number from set bin table - # - def getNewId(self): - sql_cmd = 'select max(path) from %s;' % (self._table_idx) - self._iDb.execute(sql_cmd) - max_id = self._iDb.fetchall()[0][0] - if max_id != None: - return int(max_id)+1 - else: - return 1 - - ## Get a set list instance between start and end parameters - # using the bin scheme - # - # @param seqName reference seq name - # @param start start coordinate - # @param end end coordinate - # @return lSet set list - # @note old name was getSetList_from_qcoord - # - def getSetListFromQueryCoord(self, seqName, start, end): - - min_coord = min(start,end) - max_coord = max(start,end) - - sql_cmd = 'select path from %s where contig="%s" and ('\ - % (self._table + "_idx", seqName) - for i in xrange(8, 2, -1): - bin_lvl = pow(10, i) - if int(start/bin_lvl) == int(end/bin_lvl): - bin = float(bin_lvl + (int(start / bin_lvl) / 1e10)) - sql_cmd += 'bin=%f' % (bin) - else: - bin1 = float(bin_lvl + (int(start / bin_lvl) / 1e10)) - bin2 = float(bin_lvl + (int(end /bin_lvl) / 1e10)) - sql_cmd += 'bin between %f and %f' % (bin1, bin2) - if bin_lvl != 1000: - sql_cmd += " or " - - sql_cmd += ") and min<=%d and max>=%d" % (max_coord, min_coord); - self._iDb.execute(sql_cmd) - res = self._iDb.fetchall() - lId = [] - for i in res: - lId.append(int(i[0])) - lSet = self.getSetListFromIdList(lId) - return lSet - - ## Get a set list instances strictly included between start and end parameters - # using the bin scheme - # - # @param seqName reference seq name - # @param start start coordinate - # @param end end coordinate - # @return lSet set list - # @note old name was getInSetList_from_qcoord - # @warning the implementation has been changed : I added the two first lines - # - def getSetListStrictlyIncludedInQueryCoord(self, contig, start, end): - min_coord = min(start,end) - max_coord = max(start,end) - lSet = self.getSetListFromQueryCoord(contig, start, end) - lSetStrictlyIncluded = [] - for iSet in lSet: - if iSet.getMin() > min_coord and \ - iSet.getMax() < max_coord: - lSetStrictlyIncluded.append(iSet) - - return lSetStrictlyIncluded - - ## Get a list of the identifier Id contained in the table bin - # - # @return lId list of int list of identifier - # @note old name was getSet_num - # - def getIdList(self): - sql_cmd = 'select distinct path from %s;' % (self._table_idx) - self._iDb.execute(sql_cmd) - res = self._iDb.fetchall() - lId = [] - for t in res: - lId.append(int(t[0])) - return lId - - ## Get a list of the query sequence name contained in the table bin - # - # @return lSeqName list of string list of query sequence name - # @note old name was getContig_name - # - def getSeqNameList(self): - sql_cmd = 'select distinct contig from %s;' % (self._table_idx) - self._iDb.execute(sql_cmd) - res = self._iDb.fetchall() - lSeqName = [] - for t in res: - lSeqName.append(t[0]) - return lSeqName - - ## Insert a Set list with the same new identifier in the table bin and set - # - # @note old name was insAddSetList - # - def insertListInSetAndBinTable(self, lSets, delayed = False): - id = self.getNewId() - SetUtils.changeIdInList( lSets, id ) - for iSet in lSets: - self.insASetInSetAndBinTable(iSet, delayed) - - ## Insert a set list instances In table Bin and Set and merge all overlapping sets - # - # @param lSets reference seq name - # @note old name was insMergeSetList - # - def insertListInSetAndBinTableAndMergeAllSets(self, lSets): - min, max = SetUtils.getListBoundaries(lSets) - oldLSet = self.getSetListFromQueryCoord(lSets[0].seqname, min, max) - oldQueryhash = SetUtils.getDictOfListsWithIdAsKey(oldLSet) - qhash = SetUtils.getDictOfListsWithIdAsKey(lSets) - for lNewSetById in qhash.values(): - found = False - for currentId, oldLsetById in oldQueryhash.items(): - if SetUtils.areSetsOverlappingBetweenLists(lNewSetById, oldLsetById): - oldLsetById.extend(lNewSetById) - oldLsetById = SetUtils.mergeSetsInList(oldLsetById) - self.deleteFromIdFromSetAndBinTable(currentId) - found = True - if not found: - self.insertListInSetAndBinTable(lNewSetById) - else: - id = self.getNewId() - SetUtils.changeIdInList(oldLsetById, id) - self.insertListInSetAndBinTable(oldLsetById) - - ## Insert a set list instances In table Bin and Set after removing all overlaps between database and lSets - # - # @param lSets reference seq name - # @note old name was insDiffSetList - # - def insertListInSetAndBinTableAndRemoveOverlaps(self, lSets): - min, max = SetUtils.getListBoundaries(lSets) - oldLSet = self.getSetListFromQueryCoord(lSets[0].seqname, min, max) - oldQueryHash = SetUtils.getDictOfListsWithIdAsKey(oldLSet) - newQueryHash = SetUtils.getDictOfListsWithIdAsKey(lSets) - for lNewSetById in newQueryHash.values(): - for lOldSetById in oldQueryHash.values(): - if SetUtils.areSetsOverlappingBetweenLists(lNewSetById, lOldSetById): - lNewSetById = SetUtils.getListOfSetWithoutOverlappingBetweenTwoListOfSet(lOldSetById, lNewSetById) - self.insertListInSetAndBinTable(lNewSetById)
--- a/commons/core/sql/TableJobAdaptator.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,405 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - - -import os -import time -import datetime -import sys -from commons.core.sql.Job import Job -from commons.core.sql.TableAdaptator import TableAdaptator - -## Methods for Job persistence -# -class TableJobAdaptator(TableAdaptator): - - ## Record a job - # - # @param job Job instance with the job informations - # - def recordJob(self, job): - self.removeJob(job) - sqlCmd = "INSERT INTO %s" % self._table - sqlCmd += " VALUES (" - sqlCmd += " \"%s\"," % job.jobid - sqlCmd += " \"%s\"," % job.jobname - sqlCmd += " \"%s\"," % job.groupid - sqlCmd += " \"%s\"," % job.launcher - sqlCmd += " \"%s\"," % job.queue - sqlCmd += " \"%s\"," % job.lResources - sqlCmd += " \"waiting\"," - sqlCmd += " \"%s\"," % time.strftime("%Y-%m-%d %H:%M:%S") - sqlCmd += " \"?\" );" - self._iDb.execute(sqlCmd) - - - ## Remove a job from the job table - # - # @param job: job instance to remove - # - def removeJob(self, job): - qry = "DELETE FROM %s" % self._table - qry += " WHERE groupid='%s'" % job.groupid - qry += " AND jobname='%s'" % job.jobname - qry += " AND launcher='%s';" % job.launcher - self._iDb.execute(qry) - - - ## Set the jobid of a job with the id of SGE - # - # @param job job instance - # @param jobid integer - # - def updateJobIdInDB(self, job, jobid): - #TODO: check if only one job will be updated - qry = "UPDATE %s" % self._table - qry += " SET jobid='%i'" % int(jobid) - qry += " WHERE jobname='%s'" % job.jobname - qry += " AND groupid='%s'" % job.groupid - qry += " AND launcher='%s';" % job.launcher - self._iDb.execute(qry) - - - ## Get a job status - # - # @param job: a Job instance with the job informations - # - def getJobStatus(self, job): - if job.jobid != 0 and job.jobname == "": - job.jobname = job.jobid - job.jobid = 0 - qry = "SELECT status FROM %s" % self._table - qry += " WHERE groupid='%s'" % job.groupid - qry += " AND jobname='%s'" % job.jobname - qry += " AND launcher='%s';" % job.launcher - self._iDb.execute(qry) - res = self._iDb.fetchall() - if len(res) > 1: - sys.stderr.write("ERROR while getting job status: non-unique jobs\n") - sys.stderr.flush() - sys.exit(1) - if res == None or len(res) == 0: - return "unknown" - return res[0][0] - - - ## Change a job status - # - # @param job: a Job instance with the job informations - # @param status: the new status (waiting,finished,error) - # - def changeJobStatus(self, job, status): - sqlCmd = "UPDATE %s" % self._table - sqlCmd += " SET status='%s'" % status - sqlCmd += ", node='%s'" % job.node - sqlCmd += " WHERE groupid='%s'" % job.groupid - sqlCmd += " AND jobname='%s'" % job.jobname - sqlCmd += " AND launcher='%s';" % job.launcher - self._iDb.execute(sqlCmd) - - - ## Get the number of jobs belonging to the desired groupid with the desired status. - # - # @param groupid string a group identifier to record related job series - # @param status string job status (waiting, running, finished, error) - # @return int - # - def getCountStatus(self, groupid, status): - qry = "SELECT count(jobname) FROM %s" % self._table - qry += " WHERE groupid='%s'" % groupid - qry += " AND status='%s';" % status - self._iDb.execute(qry) - res = self._iDb.fetchall() - return int(res[0][0]) - - - ## Clean all job from a job group - # - # @param groupid: a group identifier to record related job series - # - def cleanJobGroup(self, groupid): - qry = "DELETE FROM %s WHERE groupid='%s';" % (self._table, groupid) - self._iDb.execute(qry) - - - ## Check if there is unfinished job from a job group. - # - # @param groupid string a group identifier to record related job series - # - def hasUnfinishedJob(self, groupid): - qry = "SELECT * FROM %s" % self._table - qry += " WHERE groupid='%s'" % groupid - qry += " and status!='finished';" - self._iDb.execute(qry) - res = self._iDb.fetchall() - if len(res) == 0: - return False - return True - - - ## Wait job finished status from a job group. - # Job are re-launched if error (max. 3 times) - # - # @param groupid string a group identifier to record related job series - # @param checkInterval integer time laps in seconds between two checks (default = 5) - # @param maxRelaunch integer max nb of times a job in error is relaunch before exiting (default = 3) - # @param exitIfTooManyErrors boolean exit if a job is still in error above maxRelaunch (default = True) - # @param timeOutPerJob integer max nb of seconds after which one tests if a job is still in SGE or not (default = 60*60=1h) - # - def waitJobGroup(self, groupid, checkInterval=5, maxRelaunch=3, exitIfTooManyErrors=True, timeOutPerJob=60*60): - dJob2Err = {} - - # retrieve the total number of jobs belonging to the desired groupid - qry = "SELECT count(jobname) FROM %s WHERE groupid='%s';" % (self._table, groupid) - self._iDb.execute(qry) - totalNbJobs = int(self._iDb.fetchall()[0][0]) - - nbTimeOuts = 0 - - while True: - time.sleep(checkInterval) - # retrieve the finished jobs and stop if all jobs are finished - nbFinishedJobs = self.getCountStatus(groupid, "finished") - if nbFinishedJobs == totalNbJobs: - break - - # retrieve the jobs in error and relaunch them if they are in error (max. 'maxRelaunch' times) - qry = "SELECT * FROM %s" % self._table - qry += " WHERE groupid='%s'" % groupid - qry += " AND status ='error';" - self._iDb.execute(qry) - lJobsInError = self._iDb.fetchall() - for job in lJobsInError: - jobName = job[1] - if not dJob2Err.has_key(jobName): - dJob2Err[jobName] = 1 - if dJob2Err[jobName] < maxRelaunch: - print "job '%s' in error, re-submitting (%i)" % (job[1], dJob2Err[job[1]]) - sys.stdout.flush() - lResources = job[5].replace("[", "").replace("]", "").replace("'", "").split(", ") - newJob = Job(jobname=jobName, groupid=job[2], launcherFile=job[3], queue=job[4], lResources=lResources) - self.submitJob(newJob) - dJob2Err[jobName] += 1 - else: - dJob2Err[jobName] += 1 - cmd = "job '%s' in permanent error (>%i)" % (jobName, maxRelaunch) - cmd += "\ngroupid = %s" % groupid - cmd += "\nnb of jobs = %i" % totalNbJobs - cmd += "\nnb of finished jobs = %i" % self.getCountStatus(groupid, "finished") - cmd += "\nnb of waiting jobs = %i" % self.getCountStatus(groupid, "waiting") - cmd += "\nnb of running jobs = %i" % self.getCountStatus(groupid, "running") - cmd += "\nnb of jobs in error = %i" % self.getCountStatus(groupid, "error") - sys.stdout.flush() - if exitIfTooManyErrors: - self.cleanJobGroup(groupid) - sys.exit(1) - else: - checkInterval = 60 - nbTimeOuts = self._checkIfJobsTableAndJobsManagerInfoAreConsistent(nbTimeOuts, timeOutPerJob, groupid) - - - ## Submit a job to a queue and record it in job table. - # - # @param job a job instance - # @param maxNbWaitingJobs integer max nb of waiting jobs before submitting a new one (default = 10000) - # @param checkInterval integer time laps in seconds between two checks (default = 30) - # @param verbose integer (default = 0) - # - def submitJob(self, job, verbose=0, maxNbWaitingJobs=10000, checkInterval=30): - if self.getJobStatus(job) in ["waiting", "running", "finished"]: - sys.stderr.write( "WARNING: job '%s' was already submitted\n" % job.jobname) - sys.stderr.flush() - self.cleanJobGroup(job.groupid) - sys.exit(1) - - while self.getCountStatus(job.groupid, "waiting") > maxNbWaitingJobs: - time.sleep(checkInterval) - - self.recordJob(job) - cmd = self._getQsubCommand(job) - returnStatus = os.system(cmd) - - if returnStatus == 0: - fileName = "jobid.stdout" - jobidFileHandler = open(fileName, "r") - jobid = self._getJobidFromJobManager(jobidFileHandler) - if verbose > 0: - print "job '%i %s' submitted" % (jobid, job.jobname) - sys.stdout.flush() - job.jobid = jobid - jobidFileHandler.close() - self.updateJobIdInDB(job, jobid) - os.remove(fileName) - return returnStatus - - - ## Get the list of nodes where jobs of one group were executed - # - # @param groupid string a group identifier of job series - # @return lNodes list of nodes names without redundancy - # - def getNodesListByGroupId(self, groupId): - qry = "SELECT DISTINCT node FROM %s" % self._table - qry += " WHERE groupid='%s'" % groupId - self._iDb.execute(qry) - res = self._iDb.fetchall() - lNodes = [] - for resTuple in res: - lNodes.append(resTuple[0]) - return lNodes - - def checkJobTable(self): - if not self._iDb.doesTableExist(self._table): - self._iDb.createTable(self._table, "jobs") - else: - lExpFields = sorted(["jobid", "jobname", "groupid", "launcher", "queue", "resources", "status", "time", "node"]) - lObsFields = sorted(self._iDb.getFieldList(self._table)) - if lExpFields != lObsFields: - self._iDb.createTable(self._table, "jobs", overwrite = True) - - def close(self): - self._iDb.close() - - def _getJobidAndNbJob(self, jobid) : - tab = jobid.split(".") - jobid = tab[0] - tab = tab[1].split(":") - nbJob = tab[0] - return jobid, nbJob - -class TableJobAdaptatorSGE(TableJobAdaptator): - - def _checkIfJobsTableAndJobsManagerInfoAreConsistent(self, nbTimeOuts, timeOutPerJob, groupid): - # retrieve the date and time at which the oldest, still-running job was submitted - sql = "SELECT jobid,jobname,time FROM %s WHERE groupid='%s' AND status='running' ORDER BY time DESC LIMIT 1" % (self._table, groupid) - self._iDb.execute( sql ) - res = self._iDb.fetchall() - if len(res) > 0: - jobid = res[0][0] - jobname = res[0][1] - dateTimeOldestJob = res[0][2] - dateTimeCurrent = datetime.datetime.now() - # delta is time between (i) first job launched of the given groupid and still in running state and (ii) current time - delta = dateTimeCurrent - dateTimeOldestJob - # check if delta is in an interval: 0 <= delta < 1h | 1h <= delta < 2h | 2h <= delta < 3h (timeOutPerJob = 1h) - if delta.seconds >= nbTimeOuts * timeOutPerJob and delta.seconds < (nbTimeOuts+1) * timeOutPerJob: - return nbTimeOuts - # delta outside the interval: go to next interval (time out) - if delta.seconds >= (nbTimeOuts+1) * timeOutPerJob: - nbTimeOuts += 1 - # Job with 'running' status should be in qstat. Because status in DB is set at 'running' by the job launched. - if not self.isJobStillHandledBySge(jobid, jobname): - # But if not, let time for the status update (in DB), if the job finished between the query execution and now. - time.sleep( 5 ) - # If no update at 'finished', exit - #TODO: check status in DB - if not self.isJobStillHandledBySge(jobid, jobname): - msg = "ERROR: job '%s', supposedly still running, is not handled by SGE anymore" % ( jobid ) - msg += "\nit was launched the %s (> %.2f hours ago)" % ( dateTimeOldestJob, timeOutPerJob/3600.0 ) - msg += "\nthis problem can be due to:" - msg += "\n* memory shortage, in that case, decrease the size of your jobs;" - msg += "\n* timeout, in that case, decrease the size of your jobs;" - msg += "\n* node failure or database error, in that case, launch the program again or ask your system administrator." - sys.stderr.write("%s\n" % msg) - sys.stderr.flush() - self.cleanJobGroup(groupid) - sys.exit(1) - return nbTimeOuts - - ## Check if a job is still handled by SGE - # - # @param jobid string job identifier - # @param jobname string job name - # - def isJobStillHandledBySge(self, jobid, jobname): - isJobInQstat = False - qstatFile = "qstat_stdout" - cmd = "qstat > %s" % qstatFile - returnStatus = os.system(cmd) - if returnStatus != 0: - msg = "ERROR while launching 'qstat'" - sys.stderr.write( "%s\n" % msg ) - sys.exit(1) - qstatFileHandler = open(qstatFile, "r") - lLines = qstatFileHandler.readlines() - for line in lLines: - tokens = line.split() - if len(tokens) > 3 and tokens[0] == str(jobid) and tokens[2] == jobname[0:len(tokens[2])]: - isJobInQstat = True - break - qstatFileHandler.close() - os.remove(qstatFile) - return isJobInQstat - - def _getQsubCommand(self, job): - cmd = "echo '%s' | " % job.launcher - prg = "qsub" - cmd += prg - cmd += " -V" - cmd += " -N %s" % job.jobname - if job.queue != "": - cmd += " -q %s" % job.queue - cmd += " -cwd" - if job.lResources != []: - cmd += " -l \"" - cmd += " ".join(job.lResources) - cmd += "\"" - if job.parallelEnvironment != "": - cmd += " -pe " + job.parallelEnvironment - cmd += " > jobid.stdout" - return cmd - - def _getJobidFromJobManager(self, jobidFileHandler): - return int(jobidFileHandler.readline().split(" ")[2]) - - -class TableJobAdaptatorTorque(TableJobAdaptator): - - def _checkIfJobsTableAndJobsManagerInfoAreConsistent(self, nbTimeOuts, timeOutPerJob, groupid): - return nbTimeOuts - - def _getQsubCommand(self, job): - cmd = "echo '%s' | " % job.launcher - prg = "qsub" - cmd += prg - cmd += " -V" - cmd += " -d %s" % os.getcwd() - cmd += " -N %s" % job.jobname - if job.queue != "": - cmd += " -q %s" % job.queue - if job.lResources != []: - cmd += " -l \"" - cmd += " ".join(job.lResources).replace("mem_free","mem") - cmd += "\"" - cmd += " > jobid.stdout" - return cmd - - def _getJobidFromJobManager(self, jobidFileHandler): - return int(jobidFileHandler.readline().split(".")[0])
--- a/commons/core/sql/TableJobAdaptatorFactory.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,66 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - -import os -import sys -from commons.core.sql.TableJobAdaptator import TableJobAdaptatorSGE -from commons.core.sql.TableJobAdaptator import TableJobAdaptatorTorque -from commons.core.sql.JobAdaptator import JobAdaptatorSGE -from commons.core.sql.JobAdaptator import JobAdaptatorTorque - -class TableJobAdaptatorFactory(object): - - def createInstance(iDb, jobTableName): - if os.environ["REPET_JOB_MANAGER"].lower() == "sge": - iTJA = TableJobAdaptatorSGE(iDb, jobTableName) - elif os.environ["REPET_JOB_MANAGER"].lower() == "torque": - iTJA = TableJobAdaptatorTorque(iDb, jobTableName) - else: - print "ERROR: unknown jobs manager : $REPET_JOB_MANAGER = %s." % os.environ["REPET_JOB_MANAGER"] - sys.exit(1) - - return iTJA - - createInstance = staticmethod(createInstance) - - def createJobInstance(): - if os.environ["REPET_JOB_MANAGER"].lower() == "sge": - iJA = JobAdaptatorSGE() - elif os.environ["REPET_JOB_MANAGER"].lower() == "torque": - iJA = JobAdaptatorTorque() - else: - print "ERROR: unknown jobs manager : $REPET_JOB_MANAGER = %s." % os.environ["REPET_JOB_MANAGER"] - sys.exit(1) - - return iJA - - - createJobInstance = staticmethod(createJobInstance) - \ No newline at end of file
--- a/commons/core/sql/TableMapAdaptator.py Mon Apr 29 03:25:37 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,193 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - - -import sys -from commons.core.sql.TableAdaptator import TableAdaptator -from commons.core.sql.ITableMapAdaptator import ITableMapAdaptator -from commons.core.coord.Map import Map -from commons.core.coord.MapUtils import MapUtils - - -## Adaptator for Map table -# -class TableMapAdaptator( TableAdaptator, ITableMapAdaptator ): - - ## Give a list of Map instances having a given seq name - # - # @param seqName string seq name - # @return lMap list of instances - # - def getListFromSeqName( self, seqName ): - sqlCmd = "SELECT * FROM %s" % (self._table) - colum2Get, type2Get, attr2Get = self._getTypeColumAttr2Get(seqName) - sqlCmd += " WHERE " + colum2Get - sqlCmd += " = " - sqlCmd = sqlCmd + type2Get - sqlCmd = sqlCmd % "'" + attr2Get + "'" - return self._iDb.getObjectListWithSQLCmd( sqlCmd, self._getInstanceToAdapt ) - - ## Give a list of Map instances overlapping a given region - # - # @param query string query name - # @param start integer start coordinate - # @param end integer end coordinate - # @return list map instances - # - def getListOverlappingCoord(self, query, start, end): - sqlCmd = 'select * from %s where chr="%s" and ((start between least(%d,%d) and greatest(%d,%d) or end between least(%d,%d) and greatest(%d,%d)) or (least(start,end)<=least(%d,%d) and greatest(start,end)>=greatest(%d,%d))) ;' % (self._table, query, start, end, start, end, start, end, start, end, start, end, start, end) - return self._iDb.getObjectListWithSQLCmd( sqlCmd, self._getInstanceToAdapt ) - - ## Give a list of Map instances having a given sequence name - # - # @param seqName string sequence name - # @return lMap list of instances - # - def getMapListFromSeqName(self, seqName): - lMap = self.getListFromSeqName( seqName ) - return lMap - -#TODO: Check getListFromSeqName method: uses name instead of seqname -# ## Give a list of Map instances having a given sequence name from list -# # -# # @param lSeqName string sequence name list -# # @return lMap list of instances -# #</