| Previous changeset 32:3441fe98a2ba (2013-04-30) Next changeset 34:529e3e6a0954 (2013-04-30) |
|
Commit message:
Deleted selected files |
|
removed:
commons/core/launcher/JobScriptTemplate.py commons/core/launcher/JobScriptTemplateLight.py commons/core/launcher/JobScriptWithFilesCopyTemplate.py commons/core/launcher/Launcher.py commons/core/launcher/Launcher2.py commons/core/launcher/LauncherUtils.py commons/core/launcher/WriteScript.py commons/core/launcher/__init__.py commons/core/launcher/test/Test_Launcher.py commons/core/launcher/test/Test_Launcher2.py commons/core/launcher/test/Test_LauncherUtils.py commons/core/launcher/test/Test_WriteScript.py commons/core/launcher/test/__init__.py commons/core/launcher/test/expFiles/expJobScriptSQLiteWithFilesCopyTemplate.py commons/core/launcher/test/expFiles/expJobScriptTemplate.py commons/core/launcher/test/expFiles/expJobScriptTemplateLight.py commons/core/launcher/test/expFiles/expJobScriptTemplate_cmdWith2Lines.py commons/core/launcher/test/expFiles/expJobScriptWithFilesCopyTemplate.py commons/core/sql/DbFactory.py commons/core/sql/DbMySql.py commons/core/sql/DbSQLite.py commons/core/sql/ITableMapAdaptator.py commons/core/sql/ITableMatchAdaptator.py commons/core/sql/ITablePathAdaptator.py commons/core/sql/ITableSeqAdaptator.py commons/core/sql/ITableSetAdaptator.py commons/core/sql/Job.py commons/core/sql/JobAdaptator.py commons/core/sql/OldRepetDB.py commons/core/sql/RepetJob.py commons/core/sql/TableAdaptator.py commons/core/sql/TableBinPathAdaptator.py commons/core/sql/TableBinSetAdaptator.py commons/core/sql/TableJobAdaptator.py commons/core/sql/TableJobAdaptatorFactory.py commons/core/sql/TableMapAdaptator.py commons/core/sql/TableMatchAdaptator.py commons/core/sql/TablePathAdaptator.py commons/core/sql/TableSeqAdaptator.py commons/core/sql/TableSetAdaptator.py commons/core/sql/__init__.py commons/core/sql/test/TestSuite_sql.py commons/core/sql/test/Test_DbFactory.py commons/core/sql/test/Test_DbMySql.py commons/core/sql/test/Test_DbSQLite.py commons/core/sql/test/Test_F_JobAdaptator.py commons/core/sql/test/Test_F_TableJobAdaptator.py commons/core/sql/test/Test_Job.py commons/core/sql/test/Test_TableBinPathAdaptator.py commons/core/sql/test/Test_TableBinSetAdaptator.py commons/core/sql/test/Test_TableJobAdaptator.py commons/core/sql/test/Test_TableJobAdaptatorFactory.py commons/core/sql/test/Test_TableMapAdaptator.py commons/core/sql/test/Test_TableMatchAdaptator.py commons/core/sql/test/Test_TablePathAdaptator.py commons/core/sql/test/Test_TableSeqAdaptator.py commons/core/sql/test/Test_TableSetAdaptator.py commons/core/sql/test/Tst_F_RepetJob.py commons/core/sql/test/Tst_RepetJob.py commons/core/sql/test/__init__.py commons/core/stat/Stat.py commons/core/stat/__init__.py commons/core/stat/test/Test_F_Stat.py commons/core/stat/test/Test_Stat.py commons/core/stat/test/__init__.py commons/core/test/Test_LoggerFactory.py commons/core/test/__init__.py commons/core/tree/Tree.py commons/core/tree/__init__.py commons/core/tree/test/Test_Tree.py commons/core/tree/test/__init__.py commons/core/tree/test/treeTestSuite.py |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/launcher/JobScriptTemplate.py --- a/commons/core/launcher/JobScriptTemplate.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| b |
| @@ -1,95 +0,0 @@ -#!/usr/bin/env python - -import os -import sys -import time -import shutil -from commons.core.checker.RepetException import RepetException -from commons.core.sql.TableJobAdaptator import TableJobAdaptator -from commons.core.sql.DbFactory import DbFactory -from commons.core.sql.Job import Job - -try: - newDir = None - print os.uname() - beginTime = time.time() - print 'beginTime=%f' % beginTime - print "work in dir '@@tmpDir@@'" - sys.stdout.flush() - if not os.path.exists( "@@tmpDir@@" ): - raise IOError("ERROR: temporary directory '@@tmpDir@@' doesn't exist") - - minFreeGigaInTmpDir = 1 - freeSpace = os.statvfs("@@tmpDir@@") - if ((freeSpace.f_bavail * freeSpace.f_frsize) / 1073741824.0 < minFreeGigaInTmpDir): - raise RepetException("ERROR: less than %iG of free space in '@@tmpDir@@'" % minFreeGigaInTmpDir) - - os.chdir("@@tmpDir@@") - newDir = "@@groupId@@_@@jobName@@_@@time@@" - if os.path.exists(newDir): - shutil.rmtree(newDir) - os.mkdir(newDir) - os.chdir(newDir) - - iJob = Job(jobname = "@@jobName@@", groupid = "@@groupId@@", launcherFile = "@@launcher@@", node = os.getenv("HOSTNAME")) - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "@@jobTableName@@") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "running") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - - @@cmdStart@@ - if log != 0: - raise RepetException("ERROR: job returned %i" % log) - else: - print "job finished successfully" - sys.stdout.flush() - @@cmdFinish@@ - - os.chdir("..") - shutil.rmtree(newDir) - - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "@@jobTableName@@") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "finished") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - - endTime = time.time() - print 'endTime=%f' % endTime - print 'executionTime=%f' % (endTime - beginTime) - print os.uname() - sys.stdout.flush() - -except IOError, e : - print e - iJob = Job(jobname = "@@jobName@@", groupid = "@@groupId@@", launcherFile = "@@launcher@@", node = os.getenv("HOSTNAME")) - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "@@jobTableName@@") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "error") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - sys.exit(1) - -except Exception, e : - print "tmpDir is : @@tmpDir@@" - print "cDir is : @@cDir@@" - print e - if newDir != None and os.path.exists("../%s" % newDir) and not os.path.exists("@@cDir@@/%s" % newDir): - os.chdir("..") - shutil.move(newDir, "@@cDir@@/%s" % newDir) - iJob = Job(jobname = "@@jobName@@", groupid = "@@groupId@@", launcherFile = "@@launcher@@", node = os.getenv("HOSTNAME")) - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "@@jobTableName@@") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "error") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - sys.exit(1) |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/launcher/JobScriptTemplateLight.py --- a/commons/core/launcher/JobScriptTemplateLight.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| b |
| @@ -1,49 +0,0 @@ -#!/usr/bin/env python - -import os -import sys -import time -import shutil -from commons.core.checker.RepetException import RepetException -try: - newDir = None - print os.uname() - beginTime = time.time() - print 'beginTime=%f' % beginTime - print "work in dir '@@tmpDir@@'" - sys.stdout.flush() - if not os.path.exists( "@@tmpDir@@" ): - raise IOError("ERROR: temporary directory '@@tmpDir@@' doesn't exist") - - minFreeGigaInTmpDir = 1 - freeSpace = os.statvfs("@@tmpDir@@") - if ((freeSpace.f_bavail * freeSpace.f_frsize) / 1073741824.0 < minFreeGigaInTmpDir): - raise RepetException("ERROR: less than %iG of free space in '@@tmpDir@@'" % minFreeGigaInTmpDir) - - os.chdir("@@tmpDir@@") - newDir = "@@groupId@@_@@jobName@@_@@time@@" - if os.path.exists(newDir): - shutil.rmtree(newDir) - os.mkdir(newDir) - os.chdir(newDir) - - @@cmdStart@@ - if log != 0: - raise RepetException("ERROR: job returned %i" % log) - else: - print "job finished successfully" - sys.stdout.flush() - @@cmdFinish@@ - - os.chdir("..") - shutil.rmtree(newDir) - endTime = time.time() - print 'endTime=%f' % endTime - print 'executionTime=%f' % (endTime - beginTime) - print os.uname() - sys.stdout.flush() - -except IOError, e : - print e - sys.stdout.flush() - sys.exit(1) \ No newline at end of file |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/launcher/JobScriptWithFilesCopyTemplate.py --- a/commons/core/launcher/JobScriptWithFilesCopyTemplate.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| @@ -1,109 +0,0 @@ -#!/usr/bin/env python - -import os -import sys -import time -import shutil -from commons.core.checker.RepetException import RepetException -from commons.core.sql.TableJobAdaptator import TableJobAdaptator -from commons.core.sql.DbFactory import DbFactory -from commons.core.sql.Job import Job - -try: - newDir = None - print os.uname() - beginTime = time.time() - print 'beginTime=%f' % beginTime - print "work in dir '@@tmpDir@@'" - sys.stdout.flush() - if not os.path.exists("@@tmpDir@@"): - raise IOError("ERROR: temporary directory '@@tmpDir@@' doesn't exist") - - fileSize = 0 - if not os.path.exists("@@groupId@@"): - @@cmdSize@@ - freeGigaNeededInTmpDir = float(1 + fileSize) - freeSpace = os.statvfs("@@tmpDir@@") - if ((freeSpace.f_bavail * freeSpace.f_frsize) / 1073741824.0 < freeGigaNeededInTmpDir): - raise RepetException("ERROR: less than %.2fG of free space in '@@tmpDir@@'" % freeGigaNeededInTmpDir) - - os.chdir("@@tmpDir@@") - if not os.path.exists("@@groupId@@"): - try: - os.mkdir("@@groupId@@") - except OSError, e : - if e.args[0] != 17: - raise RepetException("ERROR: can't create '@@groupId@@'") - os.chdir("@@groupId@@") - @@cmdCopy@@ - else: - os.chdir("@@groupId@@") - - newDir = "@@groupId@@_@@jobName@@_@@time@@" - if os.path.exists(newDir): - shutil.rmtree(newDir) - os.mkdir(newDir) - os.chdir(newDir) - - iJob = Job(jobname = "@@jobName@@", groupid = "@@groupId@@", launcherFile = "@@launcher@@", node = os.getenv("HOSTNAME")) - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "@@jobTableName@@") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "running") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - - @@cmdStart@@ - if log != 0: - raise RepetException("ERROR: job returned %i" % log) - else: - print "job finished successfully" - sys.stdout.flush() - @@cmdFinish@@ - - os.chdir("..") - shutil.rmtree(newDir) - - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "@@jobTableName@@") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "finished") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - - endTime = time.time() - print 'endTime=%f' % endTime - print 'executionTime=%f' % (endTime - beginTime) - print os.uname() - sys.stdout.flush() - -except IOError, e : - print e - iJob = Job(jobname = "@@jobName@@", groupid = "@@groupId@@", launcherFile = "@@launcher@@", node = os.getenv("HOSTNAME")) - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "@@jobTableName@@") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "error") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - sys.exit(1) - -except Exception, e : - print "tmpDir is : @@tmpDir@@" - print "cDir is : @@cDir@@" - print e - if newDir != None and os.path.exists("../%s" % newDir) and not os.path.exists("@@cDir@@/%s" % newDir): - os.chdir("..") - shutil.move(newDir, "@@cDir@@/%s" % newDir) - iJob = Job(jobname = "@@jobName@@", groupid = "@@groupId@@", launcherFile = "@@launcher@@", node = os.getenv("HOSTNAME")) - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "@@jobTableName@@") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "error") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - sys.exit(1) |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/launcher/Launcher.py --- a/commons/core/launcher/Launcher.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| b'@@ -1,229 +0,0 @@\n-from commons.tools.CleanClusterNodesAfterRepet import CleanClusterNodesAfterRepet\n-from commons.core.stat.Stat import Stat\n-from commons.core.launcher.WriteScript import WriteScript\n-from commons.core.sql.TableJobAdaptatorFactory import TableJobAdaptatorFactory\n-from commons.core.sql.Job import Job\n-import stat\n-import os\n-import re\n-import sys\n-import time\n-import glob\n-\n-class Launcher(object):\n-\n- #TODO: remove unused parameters : query="", subject="", param="", job_table=""\n- def __init__( self, jobdb, query="", subject="", param="", cdir="",\n- tmpdir="", job_table="", queue="", groupid="", acro="X",\n- chooseTemplateWithCopy = False, chooseTemplateLight = False):\n- if jobdb.__class__.__name__ == "RepetJob":\n- self.jobdb = TableJobAdaptatorFactory.createInstance(jobdb, "jobs")\n- else:\n- self.jobdb = jobdb\n- self.jobdb.checkJobTable()\n- if cdir == "":\n- cdir = os.getcwd()\n- self.cdir = cdir\n- self.tmpdir = tmpdir\n- self.groupid = groupid\n- self.acronyme = acro\n- self._chooseTemplateWithCopy = chooseTemplateWithCopy\n- self._chooseTemplateLight = chooseTemplateLight\n- self.queue, self.lResources = self.getQueueNameAndResources(queue)\n- self._createJobInstance()\n- self._nbJobs = 0\n- \n- def getQueueNameAndResources(self, configQueue):\n- tokens = configQueue.replace("\'","").split(" ")\n- queueName = ""\n- lResources = []\n- if tokens[0] != "":\n- if re.match(".*\\.q", tokens[0]):\n- queueName = tokens[0]\n- lResources = tokens[1:]\n- else:\n- lResources = tokens\n- return queueName, lResources\n-\n- def createGroupidIfItNotExist(self):\n- if self.groupid == "":\n- self.job.groupid = str(os.getpid())\n- else:\n- self.job.groupid = self.groupid\n-\n- def beginRun( self ):\n- self.createGroupidIfItNotExist()\n- if self.jobdb.hasUnfinishedJob(self.job.groupid):\n- self.jobdb.waitJobGroup(self.job.groupid)\n- else:\n- self.jobdb.cleanJobGroup(self.job.groupid)\n-\n- ## Launch one job in parallel\n- #\n- # @param cmdStart string command-line for the job to be launched\n- # @param cmdFinish string command to retrieve result files\n- # @warning the jobname has to be defined outside from this method\n- #\n- def runSingleJob(self, cmdStart, cmdFinish = "", cmdSize = "", cmdCopy = ""):\n- if self._nbJobs == 0:\n- self._nbJobs = 1\n- pid = str(os.getpid())\n- now = time.localtime()\n- #TODO: rename ClusterLauncher_ ...\n- pyFileName = self.cdir + "/ClusterLauncher_" + self.job.groupid + "_" +\\\n- self.job.jobname + "_" + str(now[0]) + "-" + str(now[1]) +\\\n- "-" + str(now[2]) + "_" + pid + ".py"\n- self.job.launcher = pyFileName\n- \n- #TODO: to remove when refactoring is done\n- cmdStart = self._indentCmd(cmdStart)\n- cmdFinish = self._indentCmd(cmdFinish)\n- \n- iWriteScript = WriteScript(self.job, self.jobdb, self.cdir, self.tmpdir, self._chooseTemplateWithCopy, self._chooseTemplateLight)\n- iWriteScript.run(cmdStart, cmdFinish, pyFileName, cmdSize, cmdCopy)\n- os.chmod(pyFileName, stat.S_IRWXU+stat.S_IRGRP+stat.S_IXGRP+stat.S_IROTH+stat.S_IXOTH)\n- sys.stdout.flush()\n- log = self.jobdb.submitJob(self.job)\n- if log != 0:\n- print "ERROR while submitting job to the cluster"\n- sys.exit(1)\n- \n- def endRun(self, cleanNodes = False):\n- string = "waiting for %i job(s) with groupid \'%s\' (%s)" % (self._nbJobs, self.job.groupid, time.strftime("%Y-%m-%d %H:%M:%S"))\n- print string; sys.stdout.flush()\n- self.jobdb.waitJobGroup(self.job.groupid)\n- if self._nbJobs > 1:\n- '..b'()\n- return stat \n-\n- def clean( self, acronyme = "", stdout = True, stderr = True ):\n- lFileToRemove = []\n- if acronyme == "":\n- acronyme = self.acronyme \n- pattern = "ClusterLauncher*%s*.py" % ( acronyme )\n- lFileToRemove.extend(glob.glob( pattern ))\n- if stdout:\n- pattern = "%s*.o*" % ( acronyme )\n- lFileToRemove.extend(glob.glob( pattern )) \n- if stderr:\n- pattern = "%s*.e*" % ( acronyme )\n- lFileToRemove.extend(glob.glob( pattern )) \n- for file in lFileToRemove:\n- os.remove(file)\n- \n- #TODO: handle of nodesMustBeCleaned => class attribute ?\n- def runLauncherForMultipleJobs(self, acronymPrefix, lCmdsTuples, cleanMustBeDone = True, nodesMustBeCleaned = False):\n- self.beginRun()\n- print "submitting job(s) with groupid \'%s\' (%s)" % (self.job.groupid, time.strftime("%Y-%m-%d %H:%M:%S"))\n- for cmdsTuple in lCmdsTuples:\n- self._nbJobs += 1\n- self.acronyme = "%s_%s" % (acronymPrefix, self._nbJobs)\n- self.job.jobname = self.acronyme\n- if len(cmdsTuple) == 2:\n- self.runSingleJob(cmdsTuple[0], cmdsTuple[1])\n- else:\n- self.runSingleJob(cmdsTuple[0], cmdsTuple[1], cmdsTuple[2], cmdsTuple[3])\n- self._createJobInstance()\n- self.createGroupidIfItNotExist()\n- self.acronyme = acronymPrefix\n- self.endRun(nodesMustBeCleaned)\n- if cleanMustBeDone:\n- self.clean("%s_" % acronymPrefix)\n- self.jobdb.close()\n-\n- def prepareCommands(self, lCmds, lCmdStart = [], lCmdFinish = [], lCmdSize = [], lCmdCopy = []):\n- cmdStart = ""\n- for cmd in lCmdStart:\n- cmdStart += "%s\\n\\t" % cmd\n- for cmd in lCmds:\n- cmdStart += "%s\\n\\t" % cmd\n- cmdFinish = ""\n- for cmd in lCmdFinish:\n- cmdFinish += "%s\\n\\t" % cmd\n- cmdSize = ""\n- for cmd in lCmdSize:\n- cmdSize += "%s\\n\\t\\t" % cmd\n- cmdCopy = ""\n- for cmd in lCmdCopy:\n- cmdCopy += "%s\\n\\t\\t" % cmd\n- return (cmdStart, cmdFinish, cmdSize, cmdCopy)\n-\n- #TODO: to remove when refactoring is done\n- def prepareCommands_withoutIndentation(self, lCmds, lCmdStart = [], lCmdFinish = [], lCmdSize = [], lCmdCopy = []):\n- cmdStart = ""\n- for cmd in lCmdStart:\n- cmdStart += "%s\\n" % cmd\n- for cmd in lCmds:\n- cmdStart += "%s\\n" % cmd\n- cmdFinish = ""\n- for cmd in lCmdFinish:\n- cmdFinish += "%s\\n" % cmd\n- cmdSize = ""\n- for cmd in lCmdSize:\n- cmdSize += "%s\\n\\t\\t" % cmd\n- cmdCopy = ""\n- for cmd in lCmdCopy:\n- cmdCopy += "%s\\n\\t\\t" % cmd\n- return (cmdStart, cmdFinish, cmdSize, cmdCopy)\n- \n- def getSystemCommand(self, prg, lArgs):\n- systemCmd = "log = os.system(\\"" + prg \n- for arg in lArgs:\n- systemCmd += " " + arg\n- systemCmd += "\\")"\n- return systemCmd\n-\n- def cleanNodes(self):\n- iCleanClusterNodeAfterRepet = CleanClusterNodesAfterRepet()\n- iCleanClusterNodeAfterRepet.setLNodes(self.jobdb.getNodesListByGroupId(self.groupid))\n- iCleanClusterNodeAfterRepet.setTempDirectory(self.tmpdir)\n- iCleanClusterNodeAfterRepet.setPattern("%s*" % self.groupid)\n- iCleanClusterNodeAfterRepet.run()\n-\n- #TODO: to remove when refactoring is done\n- def _indentCmd(self, cmd):\n- lCmd = cmd.split("\\n")\n- cmd_Tab = "%s\\n" % lCmd[0]\n- for line in lCmd[1:-1]:\n- cmd_Tab += "\\t%s\\n" % line\n- return cmd_Tab\n- \n- def _createJobInstance(self):\n- if self.lResources == []:\n- #To have mem_free=1G:\n- self.job = Job(queue=self.queue)\n- else:\n- self.job = Job(queue=self.queue, lResources=self.lResources)\n' |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/launcher/Launcher2.py --- a/commons/core/launcher/Launcher2.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| b'@@ -1,294 +0,0 @@\n-from commons.tools.CleanClusterNodesAfterRepet import CleanClusterNodesAfterRepet\n-from commons.core.stat.Stat import Stat\n-from commons.core.launcher.WriteScript import WriteScript\n-from commons.core.sql.TableJobAdaptatorFactory import TableJobAdaptatorFactory\n-from commons.core.sql.Job import Job\n-import stat\n-import os\n-import re\n-import sys\n-import time\n-import glob\n-\n-class LauncherParameter(object):\n-\n- def __init__(self, jobDB):\n- self._jobDB = jobDB\n- \n- def getJobDB(self):\n- return self._jobDB\n-\n- def setQuery(self, query):\n- self._query = query\n-\n- def setSubject(self, subject):\n- self._subject = subject\n- \n- def setParam(self, param):\n- self._param = param\n- \n- def setCurrentDir(self, currentDir):\n- self._currentDir = currentDir\n- \n- def getCurrentDir(self):\n- return self._currentDir \n-\n- def setTempDir(self, tempDir):\n- self._tempDir = tempDir\n- \n- def getTempDir(self):\n- return self._tempDir\n- \n- def setJobTable(self, jobTable):\n- self._jobTable = jobTable\n- \n- def setQueue(self, queue):\n- self._queue = queue\n- \n- def getQueue(self):\n- return self._queue\n- \n- def setGroupId(self, groupId):\n- self._groupId = groupId\n- \n- def getGroupId(self):\n- return self._groupId\n- \n- def setAcronym(self, acronym):\n- self._acronym = acronym\n- \n- def getAcronym(self):\n- return self._acronym\n- \n- @staticmethod\n- def createParameter(jobdb, groupid, acronym):\n-\tlauncherParameter = LauncherParameter(jobdb)\n- launcherParameter.setQuery(os.getcwd())\n- launcherParameter.setSubject("")\n- launcherParameter.setParam("")\n- launcherParameter.setCurrentDir(os.getcwd())\n- launcherParameter.setTempDir(os.getcwd())\n- launcherParameter.setJobTable("")\n- launcherParameter.setQueue("")\n- launcherParameter.setGroupId(groupid)\n- launcherParameter.setAcronym(acronym)\n-\treturn launcherParameter \n-\n- \n-class Launcher2(object):\n-\n- #TODO: remove unused parameters : query="", subject="", param="", job_table=""\n- def __init__(self, iLauncherParameter):\n- jobdb = iLauncherParameter.getJobDB()\n- cdir = iLauncherParameter.getCurrentDir()\n- if jobdb.__class__.__name__ == "RepetJob":\n- self.jobdb = TableJobAdaptatorFactory.createInstance(jobdb, "jobs")\n- else:\n- self.jobdb = jobdb\n- self.jobdb.checkJobTable()\n- if cdir == "":\n- cdir = os.getcwd()\n- self.cdir = cdir\n- self.tmpdir = iLauncherParameter.getTempDir()\n- self.groupid = iLauncherParameter.getGroupId()\n- self.acronyme = iLauncherParameter.getAcronym()\n- self._chooseTemplateWithCopy = False\n- self._chooseTemplateLight = False\n- self.queue, self.lResources = self.getQueueNameAndResources(iLauncherParameter.getQueue())\n- self._createJobInstance()\n- self._nbJobs = 0\n- \n- def getQueueNameAndResources(self, configQueue):\n- tokens = configQueue.replace("\'","").split(" ")\n- queueName = ""\n- lResources = []\n- if tokens[0] != "":\n- if re.match(".*\\.q", tokens[0]):\n- queueName = tokens[0]\n- lResources = tokens[1:]\n- else:\n- lResources = tokens\n- return queueName, lResources\n-\n- def createGroupidIfItNotExist(self):\n- if self.groupid == "":\n- self.job.groupid = str(os.getpid())\n- else:\n- self.job.groupid = self.groupid\n-\n- def beginRun( self ):\n- self.createGroupidIfItNotExist()\n- if self.jobdb.hasUnfinishedJob(self.job.groupid):\n- self.jobdb.waitJobGroup(self.job.groupid)\n- else:\n- self.jobdb.cleanJobGroup(self.job.groupid)\n-\n- ## Launch one job in parallel\n- #\n-'..b'()\n- return stat \n-\n- def clean( self, acronyme = "", stdout = True, stderr = True ):\n- lFileToRemove = []\n- if acronyme == "":\n- acronyme = self.acronyme \n- pattern = "ClusterLauncher*%s*.py" % ( acronyme )\n- lFileToRemove.extend(glob.glob( pattern ))\n- if stdout:\n- pattern = "%s*.o*" % ( acronyme )\n- lFileToRemove.extend(glob.glob( pattern )) \n- if stderr:\n- pattern = "%s*.e*" % ( acronyme )\n- lFileToRemove.extend(glob.glob( pattern )) \n- for file in lFileToRemove:\n- os.remove(file)\n- \n- #TODO: handle of nodesMustBeCleaned => class attribute ?\n- def runLauncherForMultipleJobs(self, acronymPrefix, lCmdsTuples, cleanMustBeDone = True, nodesMustBeCleaned = False):\n- self.beginRun()\n- print "submitting job(s) with groupid \'%s\' (%s)" % (self.job.groupid, time.strftime("%Y-%m-%d %H:%M:%S"))\n- for cmdsTuple in lCmdsTuples:\n- self._nbJobs += 1\n- self.acronyme = "%s_%s" % (acronymPrefix, self._nbJobs)\n- self.job.jobname = self.acronyme\n- if len(cmdsTuple) == 2:\n- self.runSingleJob(cmdsTuple[0], cmdsTuple[1])\n- else:\n- self.runSingleJob(cmdsTuple[0], cmdsTuple[1], cmdsTuple[2], cmdsTuple[3])\n- self._createJobInstance()\n- self.createGroupidIfItNotExist()\n- self.acronyme = acronymPrefix\n- self.endRun(nodesMustBeCleaned)\n- if cleanMustBeDone:\n- self.clean("%s_" % acronymPrefix)\n- self.jobdb.close()\n-\n- def prepareCommands(self, lCmds, lCmdStart = [], lCmdFinish = [], lCmdSize = [], lCmdCopy = []):\n- cmdStart = ""\n- for cmd in lCmdStart:\n- cmdStart += "%s\\n\\t" % cmd\n- for cmd in lCmds:\n- cmdStart += "%s\\n\\t" % cmd\n- cmdFinish = ""\n- for cmd in lCmdFinish:\n- cmdFinish += "%s\\n\\t" % cmd\n- cmdSize = ""\n- for cmd in lCmdSize:\n- cmdSize += "%s\\n\\t\\t" % cmd\n- cmdCopy = ""\n- for cmd in lCmdCopy:\n- cmdCopy += "%s\\n\\t\\t" % cmd\n- return (cmdStart, cmdFinish, cmdSize, cmdCopy)\n-\n- #TODO: to remove when refactoring is done\n- def prepareCommands_withoutIndentation(self, lCmds, lCmdStart = [], lCmdFinish = [], lCmdSize = [], lCmdCopy = []):\n- cmdStart = ""\n- for cmd in lCmdStart:\n- cmdStart += "%s\\n" % cmd\n- for cmd in lCmds:\n- cmdStart += "%s\\n" % cmd\n- cmdFinish = ""\n- for cmd in lCmdFinish:\n- cmdFinish += "%s\\n" % cmd\n- cmdSize = ""\n- for cmd in lCmdSize:\n- cmdSize += "%s\\n\\t\\t" % cmd\n- cmdCopy = ""\n- for cmd in lCmdCopy:\n- cmdCopy += "%s\\n\\t\\t" % cmd\n- return (cmdStart, cmdFinish, cmdSize, cmdCopy)\n- \n- def getSystemCommand(self, prg, lArgs):\n- systemCmd = "log = os.system(\\"" + prg \n- for arg in lArgs:\n- systemCmd += " " + arg\n- systemCmd += "\\")"\n- return systemCmd\n-\n- def cleanNodes(self):\n- iCleanClusterNodeAfterRepet = CleanClusterNodesAfterRepet()\n- iCleanClusterNodeAfterRepet.setLNodes(self.jobdb.getNodesListByGroupId(self.groupid))\n- iCleanClusterNodeAfterRepet.setTempDirectory(self.tmpdir)\n- iCleanClusterNodeAfterRepet.setPattern("%s*" % self.groupid)\n- iCleanClusterNodeAfterRepet.run()\n-\n- #TODO: to remove when refactoring is done\n- def _indentCmd(self, cmd):\n- lCmd = cmd.split("\\n")\n- cmd_Tab = "%s\\n" % lCmd[0]\n- for line in lCmd[1:-1]:\n- cmd_Tab += "\\t%s\\n" % line\n- return cmd_Tab\n- \n- def _createJobInstance(self):\n- if self.lResources == []:\n- #To have mem_free=1G:\n- self.job = Job(queue=self.queue)\n- else:\n- self.job = Job(queue=self.queue, lResources=self.lResources)\n' |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/launcher/LauncherUtils.py --- a/commons/core/launcher/LauncherUtils.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| @@ -1,31 +0,0 @@ -class LauncherUtils(object): - - @staticmethod - def createHomogeneousSizeList(lStringSizeTuples, maxSize): - lStringSizeTuplesSorted = sorted(lStringSizeTuples, key=lambda stringSizeTuple:(stringSizeTuple[1], stringSizeTuple[0]), reverse = True) - lStringSizeList = [] - lStringSize = [] - sumTupleSize = 0 - iteratorFromBegin = 0 - iteratorFromEnd = len(lStringSizeTuplesSorted) - 1 - for tuple in lStringSizeTuplesSorted: - if sumTupleSize + tuple[1] < maxSize: - lStringSize.append(tuple[0]) - sumTupleSize += tuple[1] - elif tuple[1] >= maxSize: - lStringSizeList.append([tuple[0]]) - else: - tupleFromEnd = lStringSizeTuplesSorted[iteratorFromEnd] - while sumTupleSize + tupleFromEnd[1] < maxSize and iteratorFromBegin < iteratorFromEnd: - lStringSize.append(tupleFromEnd[0]) - sumTupleSize += tupleFromEnd[1] - del lStringSizeTuplesSorted[iteratorFromEnd] - iteratorFromEnd -= 1 - tupleFromEnd = lStringSizeTuplesSorted[iteratorFromEnd] - lStringSizeList.append(lStringSize) - lStringSize = [tuple[0]] - sumTupleSize = tuple[1] - iteratorFromBegin += 1 - if lStringSize: - lStringSizeList.append(lStringSize) - return lStringSizeList \ No newline at end of file |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/launcher/WriteScript.py --- a/commons/core/launcher/WriteScript.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| @@ -1,76 +0,0 @@ -import os -import time - -class WriteScript(object): - - def __init__(self, job = None, jobdb = None, cdir = "", tmpdir = "", chooseTemplateWithCopy = False, chooseTemplateLight = False): - self._iJob = job - self._iJobdb = jobdb - self._cDir = cdir - self._tmpDir = tmpdir - self._chooseTemplateWithCopy = chooseTemplateWithCopy - self._chooseTemplateLight = chooseTemplateLight - - def run(self, cmdStart, cmdFinish, pyFileName, cmdSize = "", cmdCopy = ""): - if self._chooseTemplateLight: - d = self.createJobScriptLightDict(cmdStart, cmdFinish, cmdSize, cmdCopy) - else: - d = self.createJobScriptDict(cmdStart, cmdFinish, cmdSize, cmdCopy) - self.fillTemplate(pyFileName, d) - - def fillTemplate(self, outputFileName, dict): - if self._chooseTemplateWithCopy: - inputFileName = "%s/commons/core/launcher/JobScriptWithFilesCopyTemplate.py" % os.environ["REPET_PATH"] - else: - inputFileName = "%s/commons/core/launcher/JobScriptTemplate.py" % os.environ["REPET_PATH"] - - if self._chooseTemplateLight: - inputFileName = "%s/commons/core/launcher/JobScriptTemplateLight.py" % os.environ["REPET_PATH"] - - input = open(inputFileName, "r") - data = input.read() - input.close() - for key, value in dict.items(): - data = data.replace("@@%s@@" % key, value) - output = open(outputFileName, "w") - output.write(data) - output.close() - - def createJobScriptDict(self, cmdStart, cmdFinish, cmdSize, cmdCopy): - dict = { - "tmpDir" : self._tmpDir, - "jobTableName" : self._iJobdb._table, - "groupId" : self._iJob.groupid, - "jobName" : self._iJob.jobname, - "launcher" : self._iJob.launcher, - "time" : time.strftime("%Y%m%d-%H%M%S"), - "repet_path" : os.environ["REPET_PATH"], - "repet_host" : os.environ["REPET_HOST"], - "repet_user" : os.environ["REPET_USER"], - "repet_pw" : os.environ["REPET_PW"], - "repet_db" : os.environ["REPET_DB"], - "repet_port" : os.environ["REPET_PORT"], - "cmdStart" : cmdStart, - "cmdFinish" : cmdFinish, - "cDir" : self._cDir, - "cmdSize" : cmdSize, - "cmdCopy" : cmdCopy - } - return dict - - def createJobScriptLightDict(self, cmdStart, cmdFinish, cmdSize, cmdCopy): - dict = { - "tmpDir" : self._tmpDir, - "jobTableName" : self._iJobdb._table, - "groupId" : self._iJob.groupid, - "jobName" : self._iJob.jobname, - "launcher" : self._iJob.launcher, - "time" : time.strftime("%Y%m%d-%H%M%S"), - "repet_path" : os.environ["REPET_PATH"], - "cmdStart" : cmdStart, - "cmdFinish" : cmdFinish, - "cDir" : self._cDir, - "cmdSize" : cmdSize, - "cmdCopy" : cmdCopy - } - return dict |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/launcher/test/Test_Launcher.py --- a/commons/core/launcher/test/Test_Launcher.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| b'@@ -1,330 +0,0 @@\n-from commons.core.utils.FileUtils import FileUtils\n-from commons.core.launcher.Launcher import Launcher\n-from commons.core.launcher.WriteScript import WriteScript\n-from commons.core.stat.Stat import Stat\n-from commons.core.sql.TableJobAdaptatorFactory import TableJobAdaptatorFactory\n-from commons.core.sql.DbFactory import DbFactory\n-from commons.core.sql.Job import Job\n-import unittest\n-import os\n-import shutil\n-import time\n-import stat\n-\n-#TODO: Test_F_Launcher.py : to execute prepareCommands() and runSingleJob()\n-# to test runLauncherForMultipleJobs()\n-#TODO: check clean of "Test_runSingleJob"\n-#TODO: refactoring => choose between "self._queue" or "lResources" to set resources\n-class Test_Launcher(unittest.TestCase):\n-\n- SARUMAN_NAME = "compute-2-46.local"\n- \n- def setUp(self):\n- self._cDir = os.getcwd()\n- self._tmpDir = self._cDir\n- self._groupid = "test"\n- self._jobTable = "dummyJobTable"\n- self._iDb = DbFactory.createInstance()\n- self._iDb.createTable(self._jobTable, "jobs", overwrite = True)\n- self._jobdb = TableJobAdaptatorFactory.createInstance(self._iDb, self._jobTable)\n- self._queue = ""\n- self._configFileName = "dummyConfigFile"\n- \n- def tearDown(self):\n- self._iDb.dropTable(self._jobTable)\n- self._iDb.close()\n- FileUtils.removeFilesByPattern(\'*.e*\')\n- FileUtils.removeFilesByPattern(\'*.o*\')\n- FileUtils.removeFilesByPattern(\'launcherFileTest_BeginRun.py\')\n- FileUtils.removeFilesByPattern(self._configFileName)\n- FileUtils.removeFilesByPattern(\'ClusterLauncher_*\')\n- \n- def test__init__wrong_fields_for_job_table(self):\n- self._iDb.dropTable(self._jobTable)\n- sqlCmd = "CREATE TABLE " + self._jobTable \n- sqlCmd += " ( jobid INT UNSIGNED"\n- sqlCmd += ", jobname VARCHAR(255)"\n- sqlCmd += ", groupid VARCHAR(255)"\n- sqlCmd += ", command TEXT"\n- sqlCmd += ", launcher VARCHAR(1024)"\n- sqlCmd += ", queue VARCHAR(255)"\n- sqlCmd += ", status VARCHAR(255)"\n- sqlCmd += ", time DATETIME"\n- sqlCmd += ", node VARCHAR(255) )"\n- self._iDb.execute(sqlCmd)\n- acronym = "Test__init__"\n- iLauncher = Launcher(self._jobdb, os.getcwd(), "", "", self._cDir, self._tmpDir, "", self._queue, self._groupid, acronym)\n- lExpFields = sorted(["jobid", "jobname", "groupid", "launcher", "queue", "resources", "status", "time", "node"])\n- lObsFields = sorted(self._iDb.getFieldList(self._jobTable))\n- self.assertEquals(lExpFields, lObsFields)\n- expJob = Job(queue = self._queue)\n- obsJob = iLauncher.job\n- self.assertEquals(expJob, obsJob)\n- \n- def test__init__withResources(self):\n- queue = "main.q mem_free=3G"\n- acronym = "Test__init__"\n- expQueue = "main.q"\n- explResources = [\'mem_free=3G\']\n- expJob = Job(queue = expQueue, lResources = explResources)\n- iLauncher = Launcher(self._jobdb, os.getcwd(), "", "", self._cDir, self._tmpDir, "", queue, self._groupid, acronym)\n- obsJob = iLauncher.job\n- self.assertEquals(expJob, obsJob)\n-\n- def test_createGroupidIfItNotExist(self):\n- acronym = "checkGroupID"\n- iLauncher = Launcher(self._jobdb, os.getcwd(), "", "", self._cDir, self._tmpDir, "", self._queue, self._groupid, acronym)\n- iLauncher.createGroupidIfItNotExist()\n- obsGroupid = iLauncher.job.groupid\n- self.assertEquals(self._groupid, obsGroupid)\n-\n- def test_createGroupidIfItNotExist_without_groupid(self):\n- groupid = ""\n- acronym = "checkGroupID"\n- iLauncher = Launcher(self._jobdb, os.getcwd(), "", "", self._cDir, self._tmpDir, "", self._queue, groupid, acronym)\n- iLauncher.createGroupidIfItNotExist()\n- obsGroupid = iLauncher.job.groupid\n- self.assertTrue(obsGroupid != "")\n- \n- def test_begi'..b'd)\n- time.sleep(20)\n- jobStatus = self._jobdb.getJobStatus(iLauncher.job)\n- os.chdir(self._cDir)\n- shutil.rmtree(acronym)\n- self.assertEqual(jobStatus, "finished")\n- \n- def test_runSingleJob_catch_error_wrong_tmpDir(self):\n- acronym = "Test_runSingleJob_catch_error"\n- os.mkdir(acronym)\n- os.chdir(acronym)\n- iLauncher = Launcher(self._jobdb, os.getcwd(), "", "", os.getcwd(), "%s/toto" % self._tmpDir, "", self._queue, self._groupid, acronym)\n- iLauncher.job.groupid = self._groupid\n- iLauncher.job.jobname = acronym\n- iLauncher.job.queue = self._queue\n- if Test_Launcher.SARUMAN_NAME == os.getenv("HOSTNAME"):\n- iLauncher.job.lResources = ["test=TRUE"]\n- cmd = "log = os.system(\\"touch \'YuFei\'\\")\\n"\n- iLauncher.runSingleJob(cmd)\n- time.sleep(20)\n- jobStatus = self._jobdb.getJobStatus(iLauncher.job) \n- os.chdir(self._cDir)\n- shutil.rmtree(acronym)\n- self.assertEqual(jobStatus, "error")\n- \n- def test_runSingleJob_catch_error_wrong_cmd(self):\n- acronym = "Test_runSingleJob_catch_error"\n- os.mkdir(acronym)\n- os.chdir(acronym)\n- iLauncher = Launcher(self._jobdb, os.getcwd(), "", "", os.getcwd(), self._tmpDir, "", self._queue, self._groupid, acronym)\n- iLauncher.job.groupid = self._groupid\n- iLauncher.job.jobname = acronym\n- iLauncher.job.queue = self._queue\n- if Test_Launcher.SARUMAN_NAME == os.getenv("HOSTNAME"):\n- iLauncher.job.lResources = ["test=TRUE"]\n- cmd = "log = os.system(\\"truc -i toto\\")\\n"\n- iLauncher.runSingleJob(cmd)\n- time.sleep(20)\n- jobStatus = self._jobdb.getJobStatus(iLauncher.job) \n- self._jobdb.cleanJobGroup(self._groupid)\n- os.chdir(self._cDir)\n- shutil.rmtree(acronym)\n- self.assertEqual(jobStatus, "error")\n-\n- def test_prepareCommands(self):\n- expCmdStart = "os.symlink(\\"../Yufei_chunks.fa\\", \\"Yufei_chunks.fa\\")\\n\\tos.symlink(\\"../Yufei_chunks.fa_cut\\", \\"Yufei_chunks.fa_cut\\")\\n\\tlog = os.system(\\"touch file\\")\\n\\t" \n- expCmdFinish = "if os.path.exists(\\"yufei.align\\"):\\n\\t\\tshutil.move(\\"yufei.align\\", \\"yufeiLuo/.\\" )\\n\\t"\n- expCmdSize = "fileSize = 3.2\\n\\t\\t"\n- expCmdCopy = "shutil.copy(\\"PY/Yufei_db/Yufei_chunks.fa\\", \\".\\")\\n\\t\\tshutil.copy(\\"PY/Yufei_db/Yufei_chunks.fa_cut\\", \\".\\")\\n\\t\\t"\n- \n- lCmdStart = []\n- lCmdStart.append("os.symlink(\\"../Yufei_chunks.fa\\", \\"Yufei_chunks.fa\\")")\n- lCmdStart.append("os.symlink(\\"../Yufei_chunks.fa_cut\\", \\"Yufei_chunks.fa_cut\\")")\n- lCmds = []\n- lCmds.append("log = os.system(\\"touch file\\")")\n- lCmdFinish = []\n- lCmdFinish.append("if os.path.exists(\\"yufei.align\\"):")\n- lCmdFinish.append("\\tshutil.move(\\"yufei.align\\", \\"yufeiLuo/.\\" )") \n- lCmdSize = []\n- lCmdSize.append("fileSize = 3.2") \n- lCmdCopy = []\n- lCmdCopy.append("shutil.copy(\\"PY/Yufei_db/Yufei_chunks.fa\\", \\".\\")")\n- lCmdCopy.append("shutil.copy(\\"PY/Yufei_db/Yufei_chunks.fa_cut\\", \\".\\")")\n-\n- iLauncher = Launcher(self._jobdb)\n- obsCmdStart, obsCmdFinish, obsCmdSize, obsCmdCopy = iLauncher.prepareCommands(lCmds, lCmdStart, lCmdFinish, lCmdSize, lCmdCopy) \n- \n- self.assertEquals(expCmdStart, obsCmdStart)\n- self.assertEquals(expCmdFinish, obsCmdFinish) \n- self.assertEquals(expCmdSize, obsCmdSize)\n- self.assertEquals(expCmdCopy, obsCmdCopy)\n- \n- def test_getSystemCommand(self):\n- prg = "touch"\n- lArgs = []\n- lArgs.append("file")\n- expCmd = "log = os.system(\\"touch file\\")"\n- iLauncher = Launcher(self._jobdb)\n- obsCmd = iLauncher.getSystemCommand(prg, lArgs)\n- self.assertEquals(expCmd, obsCmd)\n-\n-if __name__ == "__main__":\n- unittest.main()\n\\ No newline at end of file\n' |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/launcher/test/Test_Launcher2.py --- a/commons/core/launcher/test/Test_Launcher2.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| b'@@ -1,356 +0,0 @@\n-from commons.core.utils.FileUtils import FileUtils\n-from commons.core.launcher.Launcher2 import Launcher2\n-from commons.core.launcher.Launcher2 import LauncherParameter\n-from commons.core.launcher.WriteScript import WriteScript\n-from commons.core.stat.Stat import Stat\n-from commons.core.sql.TableJobAdaptatorFactory import TableJobAdaptatorFactory\n-from commons.core.sql.DbFactory import DbFactory\n-from commons.core.sql.Job import Job\n-import unittest\n-import os\n-import shutil\n-import time\n-import stat\n-\n-#TODO: Test_F_Launcher2.py : to execute prepareCommands() and runSingleJob()\n-# to test runLauncher2ForMultipleJobs()\n-#TODO: check clean of "Test_runSingleJob"\n-#TODO: refactoring => choose between "self._queue" or "lResources" to set resources\n-class Test_Launcher2(unittest.TestCase):\n-\n- SARUMAN_NAME = "compute-2-46.local"\n- \n- def setUp(self):\n- self._cDir = os.getcwd()\n- self._tmpDir = self._cDir\n- self._groupid = "test"\n- self._jobTable = "dummyJobTable"\n- self._iDb = DbFactory.createInstance()\n- self._iDb.createTable(self._jobTable, "jobs", overwrite = True)\n- self._jobdb = TableJobAdaptatorFactory.createInstance(self._iDb, self._jobTable)\n- self._queue = ""\n- self._configFileName = "dummyConfigFile"\n- \n- def tearDown(self):\n- self._iDb.dropTable(self._jobTable)\n- self._iDb.close()\n- FileUtils.removeFilesByPattern(\'*.e*\')\n- FileUtils.removeFilesByPattern(\'*.o*\')\n- FileUtils.removeFilesByPattern(\'Launcher2FileTest_BeginRun.py\')\n- FileUtils.removeFilesByPattern(self._configFileName)\n- FileUtils.removeFilesByPattern(\'ClusterLauncher2_*\')\n- \n- def test__init__wrong_fields_for_job_table(self):\n- self._iDb.dropTable(self._jobTable)\n- sqlCmd = "CREATE TABLE " + self._jobTable \n- sqlCmd += " ( jobid INT UNSIGNED"\n- sqlCmd += ", jobname VARCHAR(255)"\n- sqlCmd += ", groupid VARCHAR(255)"\n- sqlCmd += ", command TEXT"\n- sqlCmd += ", Launcher2 VARCHAR(1024)"\n- sqlCmd += ", queue VARCHAR(255)"\n- sqlCmd += ", status VARCHAR(255)"\n- sqlCmd += ", time DATETIME"\n- sqlCmd += ", node VARCHAR(255) )"\n- self._iDb.execute(sqlCmd)\n- acronym = "Test__init__"\n-\tlauncherParameter = LauncherParameter.createParameter(self._jobdb, self._groupid, acronym)\n- iLauncher2 = Launcher2(launcherParameter)\n-\n-\n- lExpFields = sorted(["jobid", "jobname", "groupid", "launcher", "queue", "resources", "status", "time", "node"])\n- lObsFields = sorted(self._iDb.getFieldList(self._jobTable))\n- self.assertEquals(lExpFields, lObsFields)\n- expJob = Job(queue = self._queue)\n- obsJob = iLauncher2.job\n- self.assertEquals(expJob, obsJob)\n- \n- def test__init__withResources(self):\n- queue = "main.q mem_free=3G"\n- acronym = "Test__init__"\n- expQueue = "main.q"\n- explResources = [\'mem_free=3G\']\n- expJob = Job(queue = expQueue, lResources = explResources)\n- \n-\tlauncherParameter = LauncherParameter.createParameter(self._jobdb, self._groupid, acronym);\n-\tlauncherParameter.setQueue(queue)\n- iLauncher2 = Launcher2(launcherParameter)\n-\n- obsJob = iLauncher2.job\n- self.assertEquals(expJob, obsJob)\n-\n- def test_createGroupidIfItNotExist(self):\n- acronym = "checkGroupID"\n-\t\n-\tlauncherParameter = LauncherParameter.createParameter(self._jobdb, self._groupid, acronym);\n- iLauncher2 = Launcher2(launcherParameter)\n- iLauncher2.createGroupidIfItNotExist()\n- obsGroupid = iLauncher2.job.groupid\n- self.assertEquals(self._groupid, obsGroupid)\n-\n- def test_createGroupidIfItNotExist_without_groupid(self):\n- groupid = ""\n- acronym = "checkGroupID"\n-\tlauncherParameter = LauncherParameter.createParameter(self._jobdb, self._groupid, acronym);\n- iLa'..b' def test_runSingleJob_catch_error_wrong_tmpDir(self):\n-# acronym = "Test_runSingleJob_catch_error"\n-# os.mkdir(acronym)\n-# os.chdir(acronym)\n-# iLauncher2= Launcher2(self._jobdb, os.getcwd(), "", "", os.getcwd(), "%s/toto" % self._tmpDir, "", self._queue, self._groupid, acronym)\n-# iLauncher2.job.groupid = self._groupid\n-# iLauncher2.job.jobname = acronym\n-# iLauncher2.job.queue = self._queue\n-# if Test_Launcher2.SARUMAN_NAME == os.getenv("HOSTNAME"):\n-# iLauncher2.job.lResources = ["test=TRUE"]\n-# cmd = "log = os.system(\\"touch \'YuFei\'\\")\\n"\n-# iLauncher2.runSingleJob(cmd)\n-# time.sleep(20)\n-# jobStatus = self._jobdb.getJobStatus(iLauncher2.job) \n-# os.chdir(self._cDir)\n-# shutil.rmtree(acronym)\n-# self.assertEqual(jobStatus, "error")\n-# \n-# def test_runSingleJob_catch_error_wrong_cmd(self):\n-# acronym = "Test_runSingleJob_catch_error"\n-# os.mkdir(acronym)\n-# os.chdir(acronym)\n-# iLauncher2 = Launcher2(self._jobdb, os.getcwd(), "", "", os.getcwd(), self._tmpDir, "", self._queue, self._groupid, acronym)\n-# iLauncher2.job.groupid = self._groupid\n-# iLauncher2.job.jobname = acronym\n-# iLauncher2.job.queue = self._queue\n-# if Test_Launcher2.SARUMAN_NAME == os.getenv("HOSTNAME"):\n-# iLauncher2.job.lResources = ["test=TRUE"]\n-# cmd = "log = os.system(\\"truc -i toto\\")\\n"\n-# iLauncher2.runSingleJob(cmd)\n-# time.sleep(20)\n-# jobStatus = self._jobdb.getJobStatus(iLauncher2.job) \n-# self._jobdb.cleanJobGroup(self._groupid)\n-# os.chdir(self._cDir)\n-# shutil.rmtree(acronym)\n-# self.assertEqual(jobStatus, "error")\n-#\n-# def test_prepareCommands(self):\n-# expCmdStart = "os.symlink(\\"../Yufei_chunks.fa\\", \\"Yufei_chunks.fa\\")\\n\\tos.symlink(\\"../Yufei_chunks.fa_cut\\", \\"Yufei_chunks.fa_cut\\")\\n\\tlog = os.system(\\"touch file\\")\\n\\t" \n-# expCmdFinish = "if os.path.exists(\\"yufei.align\\"):\\n\\t\\tshutil.move(\\"yufei.align\\", \\"yufeiLuo/.\\" )\\n\\t"\n-# expCmdSize = "fileSize = 3.2\\n\\t\\t"\n-# expCmdCopy = "shutil.copy(\\"PY/Yufei_db/Yufei_chunks.fa\\", \\".\\")\\n\\t\\tshutil.copy(\\"PY/Yufei_db/Yufei_chunks.fa_cut\\", \\".\\")\\n\\t\\t"\n-# \n-# lCmdStart = []\n-# lCmdStart.append("os.symlink(\\"../Yufei_chunks.fa\\", \\"Yufei_chunks.fa\\")")\n-# lCmdStart.append("os.symlink(\\"../Yufei_chunks.fa_cut\\", \\"Yufei_chunks.fa_cut\\")")\n-# lCmds = []\n-# lCmds.append("log = os.system(\\"touch file\\")")\n-# lCmdFinish = []\n-# lCmdFinish.append("if os.path.exists(\\"yufei.align\\"):")\n-# lCmdFinish.append("\\tshutil.move(\\"yufei.align\\", \\"yufeiLuo/.\\" )") \n-# lCmdSize = []\n-# lCmdSize.append("fileSize = 3.2") \n-# lCmdCopy = []\n-# lCmdCopy.append("shutil.copy(\\"PY/Yufei_db/Yufei_chunks.fa\\", \\".\\")")\n-# lCmdCopy.append("shutil.copy(\\"PY/Yufei_db/Yufei_chunks.fa_cut\\", \\".\\")")\n-#\n-# iLauncher2 = Launcher2(self._jobdb)\n-# obsCmdStart, obsCmdFinish, obsCmdSize, obsCmdCopy = iLauncher2.prepareCommands(lCmds, lCmdStart, lCmdFinish, lCmdSize, lCmdCopy) \n-# \n-# self.assertEquals(expCmdStart, obsCmdStart)\n-# self.assertEquals(expCmdFinish, obsCmdFinish) \n-# self.assertEquals(expCmdSize, obsCmdSize)\n-# self.assertEquals(expCmdCopy, obsCmdCopy)\n-# \n-# def test_getSystemCommand(self):\n-# prg = "touch"\n-# lArgs = []\n-# lArgs.append("file")\n-# expCmd = "log = os.system(\\"touch file\\")"\n-# iLauncher2 = Launcher2(self._jobdb)\n-# obsCmd = iLauncher2.getSystemCommand(prg, lArgs)\n-# self.assertEquals(expCmd, obsCmd)\n-\n-\n-test_suite = unittest.TestSuite()\n-test_suite.addTest( unittest.makeSuite( Test_Launcher2 ) )\n-if __name__ == "__main__":\n- unittest.TextTestRunner(verbosity=2).run( test_suite ) \n' |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/launcher/test/Test_LauncherUtils.py --- a/commons/core/launcher/test/Test_LauncherUtils.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| @@ -1,102 +0,0 @@ -import unittest -from commons.core.launcher.LauncherUtils import LauncherUtils - -class Test_LauncherUtils(unittest.TestCase): - - def test_createHomogeneousSizeList_empty(self): - lHeadersSizeTuples = [] - maxSize = 500 - expLHeadersList = [] - obsLHeadersList = LauncherUtils.createHomogeneousSizeList(lHeadersSizeTuples, maxSize) - self.assertEquals(expLHeadersList, obsLHeadersList) - - def test_createHomogeneousSizeList_one_item_upper_mean(self): - lHeadersSizeTuples = [("h1", 300)] - maxSize = 500 - expLHeadersList = [["h1"]] - obsLHeadersList = LauncherUtils.createHomogeneousSizeList(lHeadersSizeTuples, maxSize) - self.assertEquals(expLHeadersList, obsLHeadersList) - - def test_createHomogeneousSizeList_one_item_under_mean(self): - lHeadersSizeTuples = [("h1", 100)] - maxSize = 500 - expLHeadersList = [["h1"]] - obsLHeadersList = LauncherUtils.createHomogeneousSizeList(lHeadersSizeTuples, maxSize) - self.assertEquals(expLHeadersList, obsLHeadersList) - - def test_createHomogeneousSizeList_3items(self): - lHeadersSizeTuples = [("h1", 250), - ("h2", 250), - ("h3", 300)] - maxSize = 500 - expLHeadersList = [["h3"], ["h2"], ["h1"]] - obsLHeadersList = LauncherUtils.createHomogeneousSizeList(lHeadersSizeTuples, maxSize) - self.assertEquals(expLHeadersList, obsLHeadersList) - - def test_createHomogeneousSizeList_4items(self): - lHeadersSizeTuples = [("h1", 100), - ("h2", 200), - ("h3", 10), - ("h4", 400)] - maxSize = 500 - expLHeadersList = [["h4", "h3"], ["h2", "h1"]] - obsLHeadersList = LauncherUtils.createHomogeneousSizeList(lHeadersSizeTuples, maxSize) - self.assertEquals(expLHeadersList, obsLHeadersList) - - def test_createHomogeneousSizeList_5items(self): - lHeadersSizeTuples = [("h1", 300), - ("h2", 300), - ("h3", 250), - ("h4", 100), - ("h5", 90)] - maxSize = 500 - expLHeadersList = [["h2", "h5","h4"], ["h1"], ["h3"]] - obsLHeadersList = LauncherUtils.createHomogeneousSizeList(lHeadersSizeTuples, maxSize) - self.assertEquals(expLHeadersList, obsLHeadersList) - - def test_createHomogeneousSizeList_all_upper_max(self): - lHeadersSizeTuples = [("h1", 600), - ("h2", 500), - ("h3", 700), - ("h4", 900), - ("h5", 500)] - maxSize = 500 - expLHeadersList = [["h4"], ["h3"], ["h1"], ["h5"], ["h2"]] - obsLHeadersList = LauncherUtils.createHomogeneousSizeList(lHeadersSizeTuples, maxSize) - self.assertEquals(expLHeadersList, obsLHeadersList) - - def test_createHomogeneousSizeList_all_upper_mean(self): - lHeadersSizeTuples = [("h1", 300), - ("h2", 300), - ("h3", 300), - ("h4", 300), - ("h5", 300)] - maxSize = 500 - expLHeadersList = [["h5"], ["h4"], ["h3"], ["h2"], ["h1"]] - obsLHeadersList = LauncherUtils.createHomogeneousSizeList(lHeadersSizeTuples, maxSize) - self.assertEquals(expLHeadersList, obsLHeadersList) - - def test_createHomogeneousSizeList_all_under_mean(self): - lHeadersSizeTuples = [("h1", 100), - ("h2", 100), - ("h3", 100), - ("h4", 100), - ("h5", 100)] - maxSize = 500 - expLHeadersList = [["h5", "h4", "h3", "h2"], ["h1"]] - obsLHeadersList = LauncherUtils.createHomogeneousSizeList(lHeadersSizeTuples, maxSize) - self.assertEquals(expLHeadersList, obsLHeadersList) - - def test_createHomogeneousSizeList_floats(self): - lHeadersSizeTuples = [("h1", 99.1), - ("h2", 100.7), - ("h3", 100.1), - ("h4", 100.1), - ("h5", 100)] - maxSize = 500 - expLHeadersList = [['h2', 'h4', 'h3', 'h5'], ["h1"]] - obsLHeadersList = LauncherUtils.createHomogeneousSizeList(lHeadersSizeTuples, maxSize) - self.assertEquals(expLHeadersList, obsLHeadersList) - -if __name__ == "__main__": - unittest.main() \ No newline at end of file |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/launcher/test/Test_WriteScript.py --- a/commons/core/launcher/test/Test_WriteScript.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| b'@@ -1,365 +0,0 @@\n-from commons.core.utils.FileUtils import FileUtils\n-from commons.core.launcher.WriteScript import WriteScript\n-from commons.core.sql.Job import Job\n-from commons.core.sql.DbFactory import DbFactory\n-from commons.core.sql.TableJobAdaptatorFactory import TableJobAdaptatorFactory\n-import unittest\n-import os\n-import shutil\n-import time\n-import threading\n-\n-class Test_WriteScript(unittest.TestCase):\n-\n- def setUp(self):\n- self._testDir = os.getcwd()\n- self._acronym = "dummyAcronym"\n- self._jobTable = "dummyJobsTable"\n- self._iDb = DbFactory.createInstance()\n- self._iDb.createTable(self._jobTable, "jobs", overwrite = True)\n- self._jobdb = TableJobAdaptatorFactory.createInstance(self._iDb, self._jobTable)\n- self._job = Job()\n- self._job.groupid = "groupid"\n- self._job.jobname = self._acronym\n- self._job.launcher = "ClusterLauncher"\n- self._jobdb.recordJob(self._job)\n- self._dummyScratch = "dummyScratch"\n- os.mkdir(self._dummyScratch)\n- os.chdir(self._dummyScratch)\n- self._tmpDir = os.getcwd()\n- self._iScriptWriter = WriteScript(self._job, self._jobdb, self._testDir, self._tmpDir)\n- \n- def tearDown(self):\n- self._iDb.dropTable(self._jobTable)\n- self._iDb.close()\n- if FileUtils.isRessourceExists(self._dummyScratch):\n- shutil.rmtree(self._dummyScratch)\n-\n- def test_run(self):\n- isScriptAsRun = False\n- fileToCreate = \'dummyFile\'\n- cmdStart = "log = os.system( \\"touch %s\\" )\\n" % fileToCreate\n- cmdFinish = "os.system(\\"mv %s %s\\" )\\n" % (fileToCreate, self._testDir)\n- pyFileName = "%s/ClusterLauncher_%s.py" % (os.getcwd(), self._acronym) \n- \n- self._iScriptWriter.run(cmdStart, cmdFinish, pyFileName)\n- os.system("python %s" % pyFileName)\n-\n- os.chdir(self._testDir)\n- if FileUtils.isRessourceExists(fileToCreate):\n- os.remove(fileToCreate)\n- isScriptAsRun = True\n- expJobStatus = "finished" \n- obsJobStatus = self._jobdb.getJobStatus(self._job)\n- \n- self.assertTrue(isScriptAsRun)\n- self.assertEquals(expJobStatus, obsJobStatus)\n- \n- def test_run_with_cmdSize_and_cmdCopy(self):\n- isScriptAsRun = False\n- fileToCreate = \'dummyFile\'\n- fileSize = 0.5\n- cmdSize = "fileSize = %f\\n" % fileSize\n- cmdCopy = "os.system(\\"touch bank.fa\\")\\n"\n- cmdStart = "log = os.system(\\"touch %s\\")\\n" % fileToCreate\n- cmdFinish = "shutil.move(\\"%s\\", \\"%s\\")" % (fileToCreate, self._testDir)\n- pyFileName = "%s/ClusterLauncher_%s.py" % (os.getcwd(), self._acronym) \n- \n- iWriteScript = WriteScript(self._job, self._jobdb, self._testDir, self._tmpDir, True)\n- iWriteScript.run(cmdStart, cmdFinish, pyFileName, cmdSize, cmdCopy)\n- os.system("python %s" % pyFileName)\n-\n- os.chdir(self._testDir)\n- if FileUtils.isRessourceExists(fileToCreate):\n- os.remove(fileToCreate)\n- isScriptAsRun = True\n- expJobStatus = "finished" \n- obsJobStatus = self._jobdb.getJobStatus(self._job)\n- \n- self.assertTrue(isScriptAsRun)\n- self.assertEquals(expJobStatus, obsJobStatus)\n-\n-#TODO: how to test ?\n-# def test_run_2_jobs_trying_to_create_same_groupIdDir(self):\n-# fileToCreate1 = \'dummyFile1\'\n-# fileToCreate2 = \'dummyFile2\'\n-# flagFileOSError = "osErrorRaised"\n-# \n-# fileSize = 0.5\n-# cmd_checkSize = ""\n-# cmd_checkSize += "if not os.path.exists( \\"%s\\" ):\\n" % self._job.groupid\n-# cmd_checkSize += "\\tfileSize = %f\\n" % fileSize\n-# \n-# cmd_checkGroupidDir1 = ""\n-# cmd_checkGroupidDir1 += "if not os.path.exists(\\"%s\\"):\\n" % self._job.groupid\n-# cmd_checkGroupidDir1 += "\\ttry:\\n"\n-# cmd_checkGroupidDir1 += "\\t\\ttime.sleep('..b'JobsTable",\n- "groupId" : "groupid",\n- "jobName" : "job1",\n- "launcher" : "ClusterLauncher",\n- "time" : "20110505-105353",\n- "repet_path" : "/home/user/workspace/repet_pipe",\n- "cmdStart" : "log = os.system(\\"touch dummyFile1\\")",\n- "cmdFinish" : "shutil.move(\\"dummyFile1\\", \\"/home/user/workspace/repet_pipe/commons/core/launcher/test\\")",\n- "cDir" : "/home/user/workspace/repet_pipe/commons/core/launcher/test/",\n- "cmdSize" : "fileSize = 0.500000",\n- "cmdCopy" : "os.system(\\"touch bank.fa\\")"\n- }\n- expFileName = "expFiles/expJobScriptTemplateLight.py"\n- obsFileName = "obs.py"\n- \n- iWS = WriteScript(chooseTemplateLight = True)\n- iWS.fillTemplate(obsFileName, d)\n- self.assertTrue(FileUtils.are2FilesIdentical(expFileName, obsFileName))\n- os.remove(obsFileName)\n- \n- def test_createJobScriptDict(self):\n- os.chdir("..")\n- cmd_start = "log = os.system(\\"touch dummyFile1\\")"\n- cmd_finish = "shutil.move(\\"dummyFile1\\", \\"/home/user/workspace/repet_pipe/commons/core/launcher/test\\")"\n- cmd_size = ""\n- cmd_copy = ""\n- expDict = {\n- "tmpDir" : self._tmpDir,\n- "jobTableName" : self._jobTable,\n- "groupId" : self._job.groupid,\n- "jobName" : self._acronym,\n- "launcher" : self._job.launcher,\n- "time" : time.strftime("%Y%m%d-%H%M%S"),\n- "repet_path" : os.environ["REPET_PATH"],\n- "repet_host" : os.environ["REPET_HOST"],\n- "repet_user" : os.environ["REPET_USER"],\n- "repet_pw" : os.environ["REPET_PW"],\n- "repet_db" : os.environ["REPET_DB"],\n- "repet_port" : os.environ["REPET_PORT"],\n- "cmdStart" : cmd_start,\n- "cmdFinish" : cmd_finish,\n- "cDir" : self._testDir,\n- "cmdSize" : cmd_size,\n- "cmdCopy" : cmd_copy\n- }\n- obsDict = self._iScriptWriter.createJobScriptDict(cmd_start, cmd_finish, cmd_size, cmd_copy)\n- self.assertEquals(expDict, obsDict)\n- \n- def test_createJobScriptDict_with_cmdSize_and_cmdCopy(self):\n- os.chdir("..")\n- cmd_start = "log = os.system(\\"touch dummyFile1\\")"\n- cmd_finish = "shutil.move(\\"dummyFile1\\", \\"/home/user/workspace/repet_pipe/commons/core/launcher/test\\")"\n- cmd_size = "fileSize = 0.500000"\n- cmd_copy = "os.system(\\"touch bank.fa\\")"\n- expDict = {\n- "tmpDir" : self._tmpDir,\n- "jobTableName" : self._jobTable,\n- "groupId" : self._job.groupid,\n- "jobName" : self._acronym,\n- "launcher" : self._job.launcher,\n- "time" : time.strftime("%Y%m%d-%H%M%S"),\n- "repet_path" : os.environ["REPET_PATH"],\n- "repet_host" : os.environ["REPET_HOST"],\n- "repet_user" : os.environ["REPET_USER"],\n- "repet_pw" : os.environ["REPET_PW"],\n- "repet_db" : os.environ["REPET_DB"],\n- "repet_port" : os.environ["REPET_PORT"],\n- "cmdStart" : cmd_start,\n- "cmdFinish" : cmd_finish,\n- "cDir" : self._testDir,\n- "cmdSize" : cmd_size,\n- "cmdCopy" : cmd_copy\n- }\n- obsDict = self._iScriptWriter.createJobScriptDict(cmd_start, cmd_finish, cmd_size, cmd_copy)\n- self.assertEquals(expDict, obsDict)\n- \n-class CreateFileThread(threading.Thread):\n-\n- def __init__(self, pyFileName):\n- threading.Thread.__init__(self)\n- self._pyFileName = pyFileName\n- \n- def run(self):\n- os.system("python %s" % self._pyFileName)\n-\n-test_suite = unittest.TestSuite()\n-test_suite.addTest( unittest.makeSuite( Test_WriteScript ) )\n-if __name__ == "__main__":\n- unittest.TextTestRunner(verbosity=2).run( test_suite ) \n' |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/launcher/test/expFiles/expJobScriptSQLiteWithFilesCopyTemplate.py --- a/commons/core/launcher/test/expFiles/expJobScriptSQLiteWithFilesCopyTemplate.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| @@ -1,107 +0,0 @@ -#!/usr/bin/env python - -import os -import sys -import time -import shutil -from commons.core.checker.RepetException import RepetException -from commons.core.sql.TableJobAdaptator import TableJobAdaptator -from commons.core.sql.DbMySql import DbMySql -from commons.core.sql.DbSQLite import DbSQLite -from commons.core.sql.Job import Job - -try: - newDir = None - print os.uname() - beginTime = time.time() - print 'beginTime=%f' % beginTime - print "work in dir '/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch'" - sys.stdout.flush() - if not os.path.exists("/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch"): - raise IOError("ERROR: temporary directory '/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch' doesn't exist") - - fileSize = 0 - if not os.path.exists("groupid"): - fileSize = 0.500000 - freeGigaNeededInTmpDir = float(1 + fileSize) - freeSpace = os.statvfs("/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch") - if ((freeSpace.f_bavail * freeSpace.f_frsize) / 1073741824.0 < freeGigaNeededInTmpDir): - raise RepetException("ERROR: less than %.2fG of input file in '/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch'" % freeGigaNeededInTmpDir) - - os.chdir("/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch") - if not os.path.exists("groupid"): - try: - os.mkdir("groupid") - except OSError, e : - if e.args[0] != 17: - raise RepetException("ERROR: can't create 'groupid'") - os.chdir("groupid") - os.system("touch bank.fa") - else: - os.chdir("groupid") - - newDir = "groupid_job1_20110505-105353" - if os.path.exists(newDir): - shutil.rmtree(newDir) - os.mkdir(newDir) - os.chdir(newDir) - - queue = "main.q" - iJob = Job("jobs", jobname = "job1", groupid = "groupid", queue = queue, node = os.getenv("HOSTNAME")) - iDb = DbSQLite("/home/user/workspace/repet_pipe/commons/core/launcher/test/jobs") - iTJA = TableJobAdaptator(iDb, "jobs") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "running") - print "updated status: %s" % iTJA.getJobStatus(iJob) - iDb.close() - - log = os.system("touch dummyFile1") - if log != 0: - raise RepetException("ERROR: job returned %i" % log) - else: - print "job finished successfully" - shutil.move("dummyFile1", "/home/user/workspace/repet_pipe/commons/core/launcher/test") - - os.chdir("..") - shutil.rmtree(newDir) - - iDb = DbSQLite("/home/user/workspace/repet_pipe/commons/core/launcher/test/jobs") - iTJA = TableJobAdaptator(iDb, "jobs") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "finished") - print "updated status: %s" % iTJA.getJobStatus(iJob) - iDb.close() - - endTime = time.time() - print 'endTime=%f' % endTime - print 'executionTime=%f' % (endTime - beginTime) - print os.uname() - -except IOError, e : - print e - queue = "main.q" - iJob = Job("jobs", jobname = "job1", groupid = "groupid", queue = queue, node = os.getenv("HOSTNAME")) - iDb = DbSQLite("/home/user/workspace/repet_pipe/commons/core/launcher/test/jobs") - iTJA = TableJobAdaptator(iDb, "jobs") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "error") - print "updated status: %s" % iTJA.getJobStatus(iJob) - iDb.close() - sys.exit(1) - -except Exception, e : - print "tmpDir is : /home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch" - print "cDir is : /home/user/workspace/repet_pipe/commons/core/launcher/test/" - print e - if newDir != None and os.path.exists("../%s" % newDir) and not os.path.exists("/home/user/workspace/repet_pipe/commons/core/launcher/test//%s" % newDir): - os.chdir("..") - shutil.move(newDir, "/home/user/workspace/repet_pipe/commons/core/launcher/test//%s" % newDir) - queue = "main.q" - iJob = Job("jobs", jobname = "job1", groupid = "groupid", queue = queue, node = os.getenv("HOSTNAME")) - iDb = DbSQLite("/home/user/workspace/repet_pipe/commons/core/launcher/test/jobs") - iTJA = TableJobAdaptator(iDb, "jobs") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "error") - print "updated status: %s" % iTJA.getJobStatus(iJob) - iDb.close() - sys.exit(1) |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/launcher/test/expFiles/expJobScriptTemplate.py --- a/commons/core/launcher/test/expFiles/expJobScriptTemplate.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| b |
| @@ -1,95 +0,0 @@ -#!/usr/bin/env python - -import os -import sys -import time -import shutil -from commons.core.checker.RepetException import RepetException -from commons.core.sql.TableJobAdaptator import TableJobAdaptator -from commons.core.sql.DbFactory import DbFactory -from commons.core.sql.Job import Job - -try: - newDir = None - print os.uname() - beginTime = time.time() - print 'beginTime=%f' % beginTime - print "work in dir '/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch'" - sys.stdout.flush() - if not os.path.exists( "/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch" ): - raise IOError("ERROR: temporary directory '/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch' doesn't exist") - - minFreeGigaInTmpDir = 1 - freeSpace = os.statvfs("/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch") - if ((freeSpace.f_bavail * freeSpace.f_frsize) / 1073741824.0 < minFreeGigaInTmpDir): - raise RepetException("ERROR: less than %iG of free space in '/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch'" % minFreeGigaInTmpDir) - - os.chdir("/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch") - newDir = "groupid_job1_20110505-105353" - if os.path.exists(newDir): - shutil.rmtree(newDir) - os.mkdir(newDir) - os.chdir(newDir) - - iJob = Job(jobname = "job1", groupid = "groupid", launcherFile = "ClusterLauncher", node = os.getenv("HOSTNAME")) - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "dummyJobsTable") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "running") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - - log = os.system("touch dummyFile1") - if log != 0: - raise RepetException("ERROR: job returned %i" % log) - else: - print "job finished successfully" - sys.stdout.flush() - shutil.move("dummyFile1", "/home/user/workspace/repet_pipe/commons/core/launcher/test") - - os.chdir("..") - shutil.rmtree(newDir) - - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "dummyJobsTable") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "finished") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - - endTime = time.time() - print 'endTime=%f' % endTime - print 'executionTime=%f' % (endTime - beginTime) - print os.uname() - sys.stdout.flush() - -except IOError, e : - print e - iJob = Job(jobname = "job1", groupid = "groupid", launcherFile = "ClusterLauncher", node = os.getenv("HOSTNAME")) - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "dummyJobsTable") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "error") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - sys.exit(1) - -except Exception, e : - print "tmpDir is : /home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch" - print "cDir is : /home/user/workspace/repet_pipe/commons/core/launcher/test/" - print e - if newDir != None and os.path.exists("../%s" % newDir) and not os.path.exists("/home/user/workspace/repet_pipe/commons/core/launcher/test//%s" % newDir): - os.chdir("..") - shutil.move(newDir, "/home/user/workspace/repet_pipe/commons/core/launcher/test//%s" % newDir) - iJob = Job(jobname = "job1", groupid = "groupid", launcherFile = "ClusterLauncher", node = os.getenv("HOSTNAME")) - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "dummyJobsTable") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "error") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - sys.exit(1) |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/launcher/test/expFiles/expJobScriptTemplateLight.py --- a/commons/core/launcher/test/expFiles/expJobScriptTemplateLight.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| b |
| @@ -1,49 +0,0 @@ -#!/usr/bin/env python - -import os -import sys -import time -import shutil -from commons.core.checker.RepetException import RepetException -try: - newDir = None - print os.uname() - beginTime = time.time() - print 'beginTime=%f' % beginTime - print "work in dir '/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch'" - sys.stdout.flush() - if not os.path.exists( "/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch" ): - raise IOError("ERROR: temporary directory '/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch' doesn't exist") - - minFreeGigaInTmpDir = 1 - freeSpace = os.statvfs("/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch") - if ((freeSpace.f_bavail * freeSpace.f_frsize) / 1073741824.0 < minFreeGigaInTmpDir): - raise RepetException("ERROR: less than %iG of free space in '/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch'" % minFreeGigaInTmpDir) - - os.chdir("/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch") - newDir = "groupid_job1_20110505-105353" - if os.path.exists(newDir): - shutil.rmtree(newDir) - os.mkdir(newDir) - os.chdir(newDir) - - log = os.system("touch dummyFile1") - if log != 0: - raise RepetException("ERROR: job returned %i" % log) - else: - print "job finished successfully" - sys.stdout.flush() - shutil.move("dummyFile1", "/home/user/workspace/repet_pipe/commons/core/launcher/test") - - os.chdir("..") - shutil.rmtree(newDir) - endTime = time.time() - print 'endTime=%f' % endTime - print 'executionTime=%f' % (endTime - beginTime) - print os.uname() - sys.stdout.flush() - -except IOError, e : - print e - sys.stdout.flush() - sys.exit(1) \ No newline at end of file |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/launcher/test/expFiles/expJobScriptTemplate_cmdWith2Lines.py --- a/commons/core/launcher/test/expFiles/expJobScriptTemplate_cmdWith2Lines.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| b |
| @@ -1,96 +0,0 @@ -#!/usr/bin/env python - -import os -import sys -import time -import shutil -from commons.core.checker.RepetException import RepetException -from commons.core.sql.TableJobAdaptator import TableJobAdaptator -from commons.core.sql.DbFactory import DbFactory -from commons.core.sql.Job import Job - -try: - newDir = None - print os.uname() - beginTime = time.time() - print 'beginTime=%f' % beginTime - print "work in dir '/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch'" - sys.stdout.flush() - if not os.path.exists( "/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch" ): - raise IOError("ERROR: temporary directory '/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch' doesn't exist") - - minFreeGigaInTmpDir = 1 - freeSpace = os.statvfs("/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch") - if ((freeSpace.f_bavail * freeSpace.f_frsize) / 1073741824.0 < minFreeGigaInTmpDir): - raise RepetException("ERROR: less than %iG of free space in '/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch'" % minFreeGigaInTmpDir) - - os.chdir("/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch") - newDir = "groupid_job1_20110505-105353" - if os.path.exists(newDir): - shutil.rmtree(newDir) - os.mkdir(newDir) - os.chdir(newDir) - - iJob = Job(jobname = "job1", groupid = "groupid", launcherFile = "ClusterLauncher", node = os.getenv("HOSTNAME")) - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "dummyJobsTable") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "running") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - - print "Hello Yufei" - log = os.system("touch dummyFile1") - if log != 0: - raise RepetException("ERROR: job returned %i" % log) - else: - print "job finished successfully" - sys.stdout.flush() - shutil.move("dummyFile1", "/home/user/workspace/repet_pipe/commons/core/launcher/test") - - os.chdir("..") - shutil.rmtree(newDir) - - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "dummyJobsTable") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "finished") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - - endTime = time.time() - print 'endTime=%f' % endTime - print 'executionTime=%f' % (endTime - beginTime) - print os.uname() - sys.stdout.flush() - -except IOError, e : - print e - iJob = Job(jobname = "job1", groupid = "groupid", launcherFile = "ClusterLauncher", node = os.getenv("HOSTNAME")) - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "dummyJobsTable") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "error") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - sys.exit(1) - -except Exception, e : - print "tmpDir is : /home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch" - print "cDir is : /home/user/workspace/repet_pipe/commons/core/launcher/test/" - print e - if newDir != None and os.path.exists("../%s" % newDir) and not os.path.exists("/home/user/workspace/repet_pipe/commons/core/launcher/test//%s" % newDir): - os.chdir("..") - shutil.move(newDir, "/home/user/workspace/repet_pipe/commons/core/launcher/test//%s" % newDir) - iJob = Job(jobname = "job1", groupid = "groupid", launcherFile = "ClusterLauncher", node = os.getenv("HOSTNAME")) - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "dummyJobsTable") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "error") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - sys.exit(1) |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/launcher/test/expFiles/expJobScriptWithFilesCopyTemplate.py --- a/commons/core/launcher/test/expFiles/expJobScriptWithFilesCopyTemplate.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| @@ -1,109 +0,0 @@ -#!/usr/bin/env python - -import os -import sys -import time -import shutil -from commons.core.checker.RepetException import RepetException -from commons.core.sql.TableJobAdaptator import TableJobAdaptator -from commons.core.sql.DbFactory import DbFactory -from commons.core.sql.Job import Job - -try: - newDir = None - print os.uname() - beginTime = time.time() - print 'beginTime=%f' % beginTime - print "work in dir '/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch'" - sys.stdout.flush() - if not os.path.exists("/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch"): - raise IOError("ERROR: temporary directory '/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch' doesn't exist") - - fileSize = 0 - if not os.path.exists("groupid"): - fileSize = 0.500000 - freeGigaNeededInTmpDir = float(1 + fileSize) - freeSpace = os.statvfs("/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch") - if ((freeSpace.f_bavail * freeSpace.f_frsize) / 1073741824.0 < freeGigaNeededInTmpDir): - raise RepetException("ERROR: less than %.2fG of free space in '/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch'" % freeGigaNeededInTmpDir) - - os.chdir("/home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch") - if not os.path.exists("groupid"): - try: - os.mkdir("groupid") - except OSError, e : - if e.args[0] != 17: - raise RepetException("ERROR: can't create 'groupid'") - os.chdir("groupid") - os.system("touch bank.fa") - else: - os.chdir("groupid") - - newDir = "groupid_job1_20110505-105353" - if os.path.exists(newDir): - shutil.rmtree(newDir) - os.mkdir(newDir) - os.chdir(newDir) - - iJob = Job(jobname = "job1", groupid = "groupid", launcherFile = "ClusterLauncher", node = os.getenv("HOSTNAME")) - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "dummyJobsTable") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "running") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - - log = os.system("touch dummyFile1") - if log != 0: - raise RepetException("ERROR: job returned %i" % log) - else: - print "job finished successfully" - sys.stdout.flush() - shutil.move("dummyFile1", "/home/user/workspace/repet_pipe/commons/core/launcher/test") - - os.chdir("..") - shutil.rmtree(newDir) - - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "dummyJobsTable") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "finished") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - - endTime = time.time() - print 'endTime=%f' % endTime - print 'executionTime=%f' % (endTime - beginTime) - print os.uname() - sys.stdout.flush() - -except IOError, e : - print e - iJob = Job(jobname = "job1", groupid = "groupid", launcherFile = "ClusterLauncher", node = os.getenv("HOSTNAME")) - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "dummyJobsTable") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "error") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - sys.exit(1) - -except Exception, e : - print "tmpDir is : /home/user/workspace/repet_pipe/commons/core/launcher/test/dummyScratch" - print "cDir is : /home/user/workspace/repet_pipe/commons/core/launcher/test/" - print e - if newDir != None and os.path.exists("../%s" % newDir) and not os.path.exists("/home/user/workspace/repet_pipe/commons/core/launcher/test//%s" % newDir): - os.chdir("..") - shutil.move(newDir, "/home/user/workspace/repet_pipe/commons/core/launcher/test//%s" % newDir) - iJob = Job(jobname = "job1", groupid = "groupid", launcherFile = "ClusterLauncher", node = os.getenv("HOSTNAME")) - iDb = DbFactory.createInstance() - iTJA = TableJobAdaptator(iDb, "dummyJobsTable") - print "current status: %s" % iTJA.getJobStatus(iJob) - iTJA.changeJobStatus(iJob, "error") - print "updated status: %s" % iTJA.getJobStatus(iJob) - sys.stdout.flush() - iDb.close() - sys.exit(1) |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/DbFactory.py --- a/commons/core/sql/DbFactory.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| b |
| @@ -1,38 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - -from commons.core.sql.DbMySql import DbMySql - -class DbFactory (object): - - def createInstance(configFileName = "", verbosity = 1): - return DbMySql(cfgFileName = configFileName, verbosity = verbosity) - - createInstance = staticmethod(createInstance) \ No newline at end of file |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/DbMySql.py --- a/commons/core/sql/DbMySql.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| b'@@ -1,851 +0,0 @@\n-# Copyright INRA (Institut National de la Recherche Agronomique)\n-# http://www.inra.fr\n-# http://urgi.versailles.inra.fr\n-#\n-# This software is governed by the CeCILL license under French law and\n-# abiding by the rules of distribution of free software. You can use, \n-# modify and/ or redistribute the software under the terms of the CeCILL\n-# license as circulated by CEA, CNRS and INRIA at the following URL\n-# "http://www.cecill.info". \n-#\n-# As a counterpart to the access to the source code and rights to copy,\n-# modify and redistribute granted by the license, users are provided only\n-# with a limited warranty and the software\'s author, the holder of the\n-# economic rights, and the successive licensors have only limited\n-# liability. \n-#\n-# In this respect, the user\'s attention is drawn to the risks associated\n-# with loading, using, modifying and/or developing or reproducing the\n-# software by the user in light of its specific status of free software,\n-# that may mean that it is complicated to manipulate, and that also\n-# therefore means that it is reserved for developers and experienced\n-# professionals having in-depth computer knowledge. Users are therefore\n-# encouraged to load and test the software\'s suitability as regards their\n-# requirements in conditions enabling the security of their systems and/or \n-# data to be ensured and, more generally, to use and operate it in the \n-# same conditions as regards security. \n-#\n-# The fact that you are presently reading this means that you have had\n-# knowledge of the CeCILL license and that you accept its terms.\n-\n-# Exception hierarchy:\n-#\n-# StandardError\n-# |__Warning\n-# |__Error\n-# |__InterfaceError\n-# |__DatabaseError\n-# |__DataError\n-# |__OperationalError\n-# |__IntegrityError\n-# |__InternalError\n-# |__ProgrammingError\n-# |__NotSupportedError\n-\n-import os\n-import sys\n-import time\n-import ConfigParser\n-import MySQLdb\n-from MySQLdb import InterfaceError\n-from MySQLdb import OperationalError\n-from MySQLdb import InternalError\n-from MySQLdb import DatabaseError\n-from commons.core.seq.Bioseq import Bioseq\n-from commons.core.LoggerFactory import LoggerFactory\n-from commons.core.checker.RepetException import RepetException\n-from commons.core.sql.TablePathAdaptator import TablePathAdaptator\n-from commons.core.sql.TableSetAdaptator import TableSetAdaptator\n-\n-LOG_DEPTH = "repet.commons"\n-\n-TABLE_SCHEMA_DESCRIPTOR = {"map": [("name", "varchar(255)"), ("chr", "varchar(255)"), ("start", "int"), ("end", "int")],\n- "set": [("path", "int unsigned"), ("name", "varchar(255)"), ("chr", "varchar(255)"), ("start", "int"), ("end", "int")],\n- "match": [("query_name", "varchar(255)"), ("query_start", "int"), ("query_end", "int"), ("query_length", "int unsigned"), ("query_length_perc", "float"),\n- ("match_length_perc", "float"), ("subject_name", "varchar(255)"), ("subject_start", "int unsigned"), ("subject_end", "int unsigned"),\n- ("subject_length", "int unsigned"), ("subject_length_perc", "float"), ("E_value", "double"), ("score", "int unsigned"), ("identity", "float"),\n- ("path", "int unsigned")],\n- "path": [("path", "int unsigned"), ("query_name", "varchar(255)"), ("query_start", "int"), ("query_end", "int"), ("subject_name", "varchar(255)"),\n- ("subject_start", "int unsigned"), ("subject_end", "int unsigned"), ("E_value", "double"), ("score", "int unsigned"), ("identity", "float")],\n- "align": [("query_name", "varchar(255)"), ("query_start", "int"), ("query_end", "int"), ("subject_name", "varchar(255)"), ("subject_start", "int unsigned"),\n- '..b' # @param setTableName string new set table name\n- #\n- def convertMapTableIntoSetTable( self, mapTableName, setTableName ):\n- sqlCmd = "CREATE TABLE %s (path int(10) unsigned auto_increment primary key) select name, chr, start, end from %s;" % (setTableName, mapTableName)\n- self.execute(sqlCmd)\n- self.createIndex(setTableName, "set")\n- \n- \n- ## Convert an Align table into a Path table\n- #\n- # @param inAlignTable string name of the input Align table\n- # @param outPathTable string name of the output Path table\n- #\n- def convertAlignTableIntoPathTable( self, inAlignTable, outPathTable ):\n- self.createTable( outPathTable, "path", "", True )\n- sqlCmd = "SELECT * FROM %s" % ( inAlignTable )\n- self.execute( sqlCmd )\n- lResults = self.fetchall()\n- rowIndex = 0\n- for res in lResults:\n- rowIndex += 1\n- sqlCmd = "INSERT INTO %s" % ( outPathTable )\n- sqlCmd += " (path,query_name,query_start,query_end,subject_name,subject_start,subject_end,E_value,score,identity)"\n- sqlCmd += " VALUES ( \'%i\'" % ( rowIndex )\n- for i in res:\n- sqlCmd += \', "%s"\' % ( i )\n- sqlCmd += " )"\n- self.execute( sqlCmd )\n- self.updateInfoTable( outPathTable, "" )\n- \n- \n- ## Give a list of instances according to the SQL command\n- #\n- # @param SQLCmd string is a SQL command\n- # @param methodGetInstance2Adapt a getter method name. With this method you choose the type of intances contained in lObjs. See example in Test_DbMySql.py.\n- # @return lObjs list of instances\n- #\n- def getObjectListWithSQLCmd( self, SQLCmd, methodGetInstance2Adapt):\n- self.execute( SQLCmd )\n- res = self.fetchall()\n- lObjs = []\n- for t in res:\n- iObj = methodGetInstance2Adapt()\n- iObj.setFromTuple( t )\n- lObjs.append( iObj )\n- return lObjs\n- \n- \n- ## Give a list of integer according to the SQL command\n- #\n- # @param sqlCmd string is a SQL command\n- # @return lInteger integer list\n- #\n- def getIntegerListWithSQLCmd( self, sqlCmd ):\n- self.execute(sqlCmd)\n- res = self.fetchall()\n- lInteger = []\n- for t in res:\n- if t[0] != None:\n- lInteger.append(int(t[0]))\n- return lInteger\n- \n- \n- ## Give a int according to the SQL command\n- #\n- # @param sqlCmd string is a SQL command\n- # @return nb integer \n- #\n- def getIntegerWithSQLCmd( self, sqlCmd ):\n- self.execute(sqlCmd)\n- res = self.fetchall()\n- nb = res[0][0]\n- if nb == None:\n- nb = 0\n- return nb\n- \n- \n- ## Give a list of str according to the SQL command\n- #\n- # @param sqlCmd string is a SQL command\n- # @return lString str list\n- #\n- def getStringListWithSQLCmd( self, sqlCmd ):\n- self.execute(sqlCmd)\n- res = self.fetchall()\n- lString = []\n- for i in res:\n- lString.append(i[0])\n- return lString\n- \n-#TODO: use API to add indexes\n- ## Remove doublons in a given table\n- #\n- # @param table string name of a MySQL table\n- #\n- def removeDoublons( self, table ):\n- tmpTable = "%s_%s" % ( table, time.strftime("%Y%m%d%H%M%S") )\n- sqlCmd = "CREATE TABLE %s SELECT DISTINCT * FROM %s" % ( tmpTable, table )\n- self.execute( sqlCmd )\n- self.dropTable( table )\n- self.renameTable(tmpTable, table)\n- \n- \n- ## Get a list of table names from a pattern\n- #\n- # @note for instance pattern = \'MyProject_%\'\n- #\n- def getTableListFromPattern( self, pattern ):\n- if pattern == "*" or pattern == "%":\n- sqlCmd = "SHOW TABLES"\n- else:\n- sqlCmd = "SHOW TABLES like \'%s\'" % ( pattern )\n- lTables = self.getStringListWithSQLCmd( sqlCmd )\n- return lTables\n' |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/DbSQLite.py --- a/commons/core/sql/DbSQLite.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| @@ -1,173 +0,0 @@ -import sqlite3 -import os -import sys - -#TODO: update...compare with DbMySql.py -class DbSQLite(object): - - ## Constructor - # - # @param host string db file path - # @param cfgFileName string configuration file name - # - # @note when a parameter is left blank, the constructor is able - # to set attribute values from environment variable: REPET_HOST, - # - def __init__(self, host = ""): - if host != "": - self.host = host - else: - msg = "ERROR: no host specified" - sys.stderr.write( "%s\n" % msg ) - sys.exit(1) - # remove open() and cursor from init() use directly outside this class ... - self.open() - self.cursor = self.db.cursor() - - ## Connect to the DbSQLite database - # - # @param verbose integer (default = 0) - # - def open( self, verbose = 0, nb = 0 ): - try: - #sqlite.connect(":memory:", check_same_thread = False) - self.db = sqlite3.connect(self.host, check_same_thread= False, isolation_level=None, detect_types=sqlite3.PARSE_DECLTYPES) - except sqlite3.Error, e: - if verbose > 0: - print "ERROR %s" % e - sys.stdout.flush() - return False - return True - - ## Execute a SQL query - # - # @param qry string SQL query to execute - # @param params parameters of SQL query - # - def execute( self, qry, params=None ): - try : - if params == None: - self.cursor.execute( qry ) - else: - self.cursor.execute( qry, params ) - except Exception, e: - #TODO Must be test - try : - if params == None: - self.cursor.execute( qry ) - else: - self.cursor.execute( qry, params ) - except Exception, e: - print "Erreur : %s" % e - - ## Retrieve the results of a SQL query - # - def fetchall(self): - return self.cursor.fetchall() - - ## Record a new table in the 'info_table' table - # - # @param tableName string table name - # @param info string information on the table origin - # - def updateInfoTable( self, tableName, info ): - if not self.doesTableExist( "info_tables" ): - sqlCmd = "CREATE TABLE info_tables ( name varchar(255), file varchar(255) )" - self.execute( sqlCmd ) - sqlCmd = 'INSERT INTO info_tables VALUES ("%s","%s")' % (tableName, info) - self.execute( sqlCmd ) - - def createTable(self, tableName, dataType, overwrite=False, verbose=0): - if verbose > 0: - print "creating table '%s' from file '%s' of type '%s'..." % (tableName, dataType) - sys.stdout.flush() - if overwrite: - self.dropTable(tableName) - if dataType.lower() in ["job", "jobs"]: - self.createJobTable(tableName) - else: - print "ERROR: unknown type %s" % (dataType) - self.close() - sys.exit(1) - if verbose > 0: - print "done!"; sys.stdout.flush() - - ## Create a job table - # - # @param tablename new table name - # - def createJobTable( self, tablename ): - sqlCmd = "CREATE TABLE %s" % ( tablename ) - sqlCmd += " ( jobid INT UNSIGNED" - sqlCmd += ", jobname VARCHAR(255)" - sqlCmd += ", groupid VARCHAR(255)" - sqlCmd += ", command TEXT" - sqlCmd += ", launcher VARCHAR(1024)" - sqlCmd += ", queue VARCHAR(255)" - sqlCmd += ", status VARCHAR(255)" - sqlCmd += ", time timestamp" - sqlCmd += ", node VARCHAR(255) )" - self.execute( sqlCmd ) - - self.updateInfoTable( tablename, "job table" ) - sqlCmd = "CREATE INDEX igroupid ON " + tablename + " ( groupid )" - self.execute( sqlCmd ) - - ## Test if a table exists - # - # @param table string table name - # @return boolean True if the table exists, False otherwise - # - def doesTableExist( self, table ): - qry = "PRAGMA table_info(%s)" % (table) - self.execute( qry ) - results = self.cursor.fetchall() - if results: - return True - return False - - def isEmpty( self, tableName ): - return self.getSize( tableName ) == 0 - - ## Give the rows number of the table - # - # @param tableName string table name - # - def getSize( self, tableName ): - qry = "SELECT count(*) FROM %s;" % ( tableName ) - self.execute( qry ) - res = self.fetchall() - return int( res[0][0] ) - - ## Remove a table if it exists - # - # @param table string table name - # @param verbose integer (default = 0) - # - def dropTable( self, table, verbose = 0 ): - if self.doesTableExist( table ): - sqlCmd = "DROP TABLE %s" % ( table ) - self.execute( sqlCmd ) - sqlCmd = 'DELETE FROM info_tables WHERE name = "%s"' % ( table ) - self.execute( sqlCmd ) - - ## Get a list with the fields - # - def getFieldList( self, table ): - lFields = [] - sqlCmd = "PRAGMA table_info(%s)" % ( table ) - self.execute( sqlCmd ) - lResults = self.fetchall() - for res in lResults: - lFields.append( res[1] ) - return lFields - - ## delete this SQLite database session - # - def delete(self): - os.remove(self.host) - - ## Close the connection - # - def close( self ): - self.db.close() |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/ITableMapAdaptator.py --- a/commons/core/sql/ITableMapAdaptator.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| b |
| @@ -1,113 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - - -## Interface for TableMapAdaptator -# -class ITableMapAdaptator(object): - - ## Insert a map instance - # - # @param obj map or set - # @param delayed boolean must the insert be delayed - # - # @warning old name was insAMap - # - def insert(self, obj, delayed=False): - pass - - - ## Insert a list of Map or Set or Match instances - # - # @param l a list of object instances - # @param delayed boolean - # - # @warning old name was insMapList - # - def insertList(self, l, delayed = False): - pass - - ## Give a list of the distinct seqName/chr present in the table - # - # @return lDistinctContigNames string list - # - # @warning old name was getContig_name - # - def getSeqNameList(self): - pass - - - ## Give a list of Map instances having a given seq name - # - # @param seqName string seq name - # @return lMap list of instances - # - # @warning old name was get_MapList_from_contig - # - def getMapListFromSeqName(self, seqName): - pass - - - ## Return a list of Set instances from a given sequence name - # - # @param seqName string sequence name - # @return lSets list of Set instances - # - # @warning old name was getSetList_from_contig - # - def getSetListFromSeqName( self, seqName ): - pass - - - ## Give a map instances list overlapping a given region - # - # @param seqName string seq name - # @param start integer start coordinate - # @param end integer end coordinate - # @return lMap list of map instances - # - # @warning old name was getMapList_from_qcoord - # - def getMapListOverlappingCoord(self, seqName, start, end): - pass - - - ## Return a list of Set instances overlapping a given region - # - # @param seqName string sequence name - # @param start integer start coordinate - # @param end integer end coordinate - # @return lSet list of Set instances - # - # @warning old name was getSetList_from_qcoord - # - def getSetListOverlappingCoord( self, seqName, start, end ): - pass - \ No newline at end of file |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/ITableMatchAdaptator.py --- a/commons/core/sql/ITableMatchAdaptator.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| b |
| @@ -1,68 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - - -## Interface for TableMatchAdaptator -# -class ITableMatchAdaptator(object): - - ## Give a list of Match instances given a query name - # - # @param query string sequence name - # @return lMatches list of Match instances - # - def getMatchListFromQuery( self, query ): - pass - - ## Give a list of Match instances having the same identifier - # - # @param id integer identifier number - # @return lMatch a list of Match instances - # - def getMatchListFromId( self, id ): - pass - - ## Insert a Match instance - # - # @param iMatch a Match instance - # @param delayed boolean - # - def insert(self, iMatch, delayed = False): - pass - - ## Insert a list of Map or Set or Match instances - # - # @param l a list of object instances - # @param delayed boolean - # - # @warning old name was insMapList - # - def insertList(self, l, delayed = False): - pass \ No newline at end of file |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/ITablePathAdaptator.py --- a/commons/core/sql/ITablePathAdaptator.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| b |
| b'@@ -1,429 +0,0 @@\n-# Copyright INRA (Institut National de la Recherche Agronomique)\n-# http://www.inra.fr\n-# http://urgi.versailles.inra.fr\n-#\n-# This software is governed by the CeCILL license under French law and\n-# abiding by the rules of distribution of free software. You can use, \n-# modify and/ or redistribute the software under the terms of the CeCILL\n-# license as circulated by CEA, CNRS and INRIA at the following URL\n-# "http://www.cecill.info". \n-#\n-# As a counterpart to the access to the source code and rights to copy,\n-# modify and redistribute granted by the license, users are provided only\n-# with a limited warranty and the software\'s author, the holder of the\n-# economic rights, and the successive licensors have only limited\n-# liability. \n-#\n-# In this respect, the user\'s attention is drawn to the risks associated\n-# with loading, using, modifying and/or developing or reproducing the\n-# software by the user in light of its specific status of free software,\n-# that may mean that it is complicated to manipulate, and that also\n-# therefore means that it is reserved for developers and experienced\n-# professionals having in-depth computer knowledge. Users are therefore\n-# encouraged to load and test the software\'s suitability as regards their\n-# requirements in conditions enabling the security of their systems and/or \n-# data to be ensured and, more generally, to use and operate it in the \n-# same conditions as regards security. \n-#\n-# The fact that you are presently reading this means that you have had\n-# knowledge of the CeCILL license and that you accept its terms.\n-\n-\n-## Interface for TablePathAdaptator\n-#\n-class ITablePathAdaptator (object):\n-\n- ## Give the data contained in the table as a list of Path instances\n- #\n- # @return lPaths list of path instances\n- #\n- def getListOfAllPaths( self ):\n- pass\n- \n- ## Give a list of Path instances having the same identifier\n- #\n- # @param id integer identifier number\n- # @return lPath a list of Path instances\n- #\n- # @warning old name was getPathList_from_num\n- #\n- def getPathListFromId( self, id ):\n- pass\n-\n- ## Give a list of Path instances according to the given list of identifier numbers\n- #\n- # @param lId integer list \n- # @return lPath a list of Path instances\n- #\n- # @warning old name was getPathList_from_numlist\n- #\n- def getPathListFromIdList( self, lId ):\n- pass\n- \n- ## Give a list of Path instances having the same given query name\n- #\n- # @param query string name of the query \n- # @return lPath a list of Path instances\n- #\n- # @warning old name was getPathList_from_query\n- #\n- def getPathListFromQuery( self, query ):\n- pass\n- \n- ## Give a list with all the distinct identifiers corresponding to the query\n- #\n- # @param query string name of the query \n- # @return lId a list of integer\n- #\n- # @warning old name was getPathList_from_query\n- #\n- def getIdListFromQuery( self, query ):\n- pass\n- \n- ## Give a list with all the distinct identifiers corresponding to the subject\n- #\n- # @param subject string name of the subject \n- # @return lId a list of integer\n- #\n- # @warning old name was getPathList_from_subject\n- #\n- def getIdListFromSubject( self, subject ):\n- pass\n- \n- ## Insert a path instance\n- #\n- # @param obj a path instance\n- # @param delayed boolean indicating if the insert must be delayed\n- #\n- # @note data are inserted such that the query is always on the direct strand\n- #\n- # @warning old name was insAPath\n- #\n- def insert(self, obj, delayed = False):\n- pass\n- \n- ## Insert a list of Path instances\n- #\n- # @param l a list of Path instances\n- # @param delayed boolean\n- #\n- # @warning old name was insPathList\n- #\n- def insertList(self, l, delayed = False):\n- pass\n- \n- ## '..b'th_from_subject\n- # \n- def getCumulLengthFromSubject( self, subjectName ):\n- pass\n- \n- ## Give a list of the length of all chains of paths for a given subject name\n- #\n- # @param subjectName string name of the subject\n- # @return lChainLengths list of lengths per chain of paths\n- # @warning doesn\'t take into account the overlaps !!\n- # @warning old name was getListChainLength_from_subject\n- #\n- def getChainLengthListFromSubject( self, subjectName ):\n- pass\n-\n- ## Give a list of identity of all chains of paths for a given subject name\n- #\n- # @param subjectName string name of the subject\n- # @return lChainIdentities list of identities per chain of paths\n- # @warning doesn\'t take into account the overlaps !!\n- # @warning old name was getListChainIdentity_from_subject\n- # \n- def getChainIdentityListFromSubject( self, subjectName ):\n- pass\n- \n- ## Give a list of Path lists sorted by weighted identity.\n- #\n- # @param qry query name\n- # @return lChains list of chains\n- #\n- def getListOfChainsSortedByAscIdentityFromQuery( self, qry ):\n- pass\n- \n- ## Give a list of the length of all paths for a given subject name\n- #\n- # @param subjectName string name of the subject\n- # @return lPathLengths list of lengths per path\n- # @warning doesn\'t take into account the overlaps !!\n- # @warning old name was getListPathLength_from_subject\n- #\n- def getPathLengthListFromSubject( self, subjectName ):\n- pass\n- \n- ## Give a a list with all distinct identifiers for a given subject sorted in decreasing order according to the length of the chains\n- # \n- # @return lPathNums a list of paths Id\n- #\n- # @warning old name was getPathNumListSortedByDecreasingChainLengthFromSubject\n- #\n- def getIdListSortedByDecreasingChainLengthFromSubject( self, subjectName ):\n- pass\n- \n- ## Give a list of Set instance list from the path contained on a query name\n- #\n- # @param query string query name\n- # @return lSet list of set instance \n- #\n- # @warning old name was getSetList_from_contig\n- #\n- def getSetListFromQuery(self, query):\n- pass\n- \n- ## Delete path corresponding to a given identifier number\n- #\n- # @param id integer identifier number\n- #\n- # @warning old name was delPath_from_num\n- #\n- def deleteFromId(self,id):\n- pass\n- \n- ## Delete path corresponding to a given list of identifier number\n- #\n- # @param lId list of identifier number\n- #\n- # @warning old name was delPath_from_numlist\n- #\n- def deleteFromIdList(self,lId):\n- pass\n-\n- ## Join two path by changing id number of id1 and id2 path to the least of id1 and id2\n- #\n- # @param id1 integer path number\n- # @param id2 integer path number\n- # @return newId integer id used to join\n- #\n- # @warning old name was joinPath\n- #\n- def joinTwoPaths(self,id1,id2):\n- pass\n- \n- ## Get a new id number\n- #\n- # @return newId integer new id\n- #\n- def getNewId(self):\n- pass\n- \n- ## Test if table is empty\n- # \n- def isEmpty( self ):\n- pass\n- \n- ## Create a \'pathRange\' table from a \'path\' table. \n- # The output table summarizes the information per identifier. \n- # The min and max value are taken. \n- # The identity is averaged over the fragments. \n- # It may overwrite an existing table.\n- #\n- # @param outTable string name of the output table\n- # @return outTable string Table which summarizes the information per identifier\n- #\n- def path2PathRange( self, outTable="" ):\n- pass\n- \n- ## Return the number of times a given instance is present in the table\n- # The identifier is not considered,\n- # only coordinates, score, E-value and identity.\n- #\n- # @return nbOcc integer\n- #\n- def getNbOccurrences( self, iPath ):\n- pass\n' |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/ITableSeqAdaptator.py --- a/commons/core/sql/ITableSeqAdaptator.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| b |
| @@ -1,63 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - - -## Interface for TableSeqAdaptator -# -class ITableSeqAdaptator(object): - - ## Retrieve all the distinct accession names in a list. - # - # @return lAccessions list of accessions - # - # @warning old name was getListAccession - # - def getAccessionsList( self ): - pass - - ## Save sequences in a fasta file from a list of accession names. - # - # @param lAccessions list of accessions - # @param outFileName string Fasta file - # - # @warning old name saveListAccessionInFastaFile - # - def saveAccessionsListInFastaFile( self, lAccessions, outFileName ): - pass - - ## insert bioseq instance - # - # @param seq bioseq - # @param delayed boolean must the insert be delayed - # - # @warning old name was insASeq - # - def insert(self, seq, delayed = False): - pass \ No newline at end of file |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/ITableSetAdaptator.py --- a/commons/core/sql/ITableSetAdaptator.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| b |
| @@ -1,146 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - -## Interface for TableSetAdaptator -# -class ITableSetAdaptator (object): - - ## Insert a set instance - # - # @param obj a set instance - # @param delayed boolean indicating if the insert must be delayed - # - # @warning old name was insASet - # - def insert(self, obj, delayed = False): - pass - - ## Insert a list of Set instances - # - # @param l a list of object instances - # @param delayed boolean - # - # @warning old name was insSetList - # - def insertList(self, l, delayed = False): - pass - - ## Give a list of identifier numbers contained in the table - # - # @return l integer list - # - # @warning old name was getSet_num - # - def getIdList(self): - pass - - ## Give a list of Set instances having a given seq name - # - # @param seqName string seq name - # @return lSets list of instances - # - # @warning old name was get_SetList_from_contig - # - def getSetListFromSeqName(self, seqName): - pass - - ## Give a set instances list with a given identifier number - # - # @param id integer identifier number - # @return lSet list of set instances - # - # @warning old name was getSetList_from_num - # - def getSetListFromId(self, id): - pass - - ## Give a set instances list with a list of identifier numbers - # - # @param lId integers list identifiers list numbers - # @return lSet list of set instances - # - # @warning old name was getSetList_from_numlist - # - def getSetListFromIdList(self,lId): - pass - - ## Return a list of Set instances overlapping a given sequence - # - # @param seqName string sequence name - # @param start integer start coordinate - # @param end integer end coordinate - # @return lSet list of Set instances - # - # @warning old name was getSetList_from_qcoord - # - def getSetListOverlappingCoord( self, seqName, start, end ): - pass - - ## Delete set corresponding to a given identifier number - # - # @param id integer identifier number - # - # @warning old name was delSet_from_num - # - def deleteFromId(self, id): - pass - - ## Delete set corresponding to a given list of identifier number - # - # @param lId integers list list of identifier number - # - # @warning old name was delSet_from_listnum - # - def deleteFromIdList(self, lId): - pass - - ## Join two set by changing id number of id1 and id2 set to the least of id1 and id2 - # - # @param id1 integer id path number - # @param id2 integer id path number - # - # @warning old name was joinSet - # - def joinTwoSets(self, id1, id2): - pass - - ## Get a new id number - # - # @return new_id integer max_id + 1 - # - def getNewId(self): - pass - - ## Give the data contained in the table as a list of Sets instances - # - # @return lSets list of set instances - # - def getListOfAllSets( self ): - pass \ No newline at end of file |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/Job.py --- a/commons/core/sql/Job.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| @@ -1,74 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - -## Job informations to launch a command on a cluster. -# -class Job(object): - - ## Constructor - # - # @param jobid the job identifier - # @param jobname the job name - # @param groupid the group identifier to record related job series - # @param queue queue name of the job manager - # @param command command launched - # @param node cluster node name where the execution takes place - # @param launcherFile file name launched as job - # @param lResources resources (memory, time...) but need to conform to SGE/Torque syntax ! - # - def __init__(self, jobid=0, jobname="", groupid="", queue="", command="", launcherFile="",\ - node="", lResources=["mem_free=1G"], parallelEnvironment="" ): - if str(jobid).isdigit(): - self.jobid = int(jobid) - self.jobname = jobname - else: - self.jobname = jobid - self.jobid = 0 - self.jobid = jobid - self.groupid = groupid - self.setQueue(queue) - self.command = command - self.launcher = launcherFile - self.node = node - self.lResources = lResources - self.parallelEnvironment = parallelEnvironment - - def setQueue(self, queue): - self.queue = "" - if queue != "none": - self.queue = queue - - def __eq__(self, o): - if self.jobid == o.jobid and self.jobname == o.jobname\ - and self.groupid == o.groupid and self.queue == o.queue and self.command == o.command \ - and self.launcher == o.launcher and self.node == o.node and self.lResources == o.lResources \ - and self.parallelEnvironment == o.parallelEnvironment: - return True - return False |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/JobAdaptator.py --- a/commons/core/sql/JobAdaptator.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| b'@@ -1,271 +0,0 @@\n-# Copyright INRA (Institut National de la Recherche Agronomique)\n-# http://www.inra.fr\n-# http://urgi.versailles.inra.fr\n-#\n-# This software is governed by the CeCILL license under French law and\n-# abiding by the rules of distribution of free software. You can use, \n-# modify and/ or redistribute the software under the terms of the CeCILL\n-# license as circulated by CEA, CNRS and INRIA at the following URL\n-# "http://www.cecill.info". \n-#\n-# As a counterpart to the access to the source code and rights to copy,\n-# modify and redistribute granted by the license, users are provided only\n-# with a limited warranty and the software\'s author, the holder of the\n-# economic rights, and the successive licensors have only limited\n-# liability. \n-#\n-# In this respect, the user\'s attention is drawn to the risks associated\n-# with loading, using, modifying and/or developing or reproducing the\n-# software by the user in light of its specific status of free software,\n-# that may mean that it is complicated to manipulate, and that also\n-# therefore means that it is reserved for developers and experienced\n-# professionals having in-depth computer knowledge. Users are therefore\n-# encouraged to load and test the software\'s suitability as regards their\n-# requirements in conditions enabling the security of their systems and/or \n-# data to be ensured and, more generally, to use and operate it in the \n-# same conditions as regards security. \n-#\n-# The fact that you are presently reading this means that you have had\n-# knowledge of the CeCILL license and that you accept its terms.\n-\n-\n-import os\n-import time\n-import sys\n-import tempfile\n-import subprocess\n-from commons.core.sql.Job import Job\n-\n-## Methods for Job persistence \n-#\n-class JobAdaptator(object):\n- \n- def __init__(self, lJob = [], table = "" ):\n- self._lJobID = lJob\n- self._table = table\n- self._acronym = ""\n- ## Record a job\n- #\n- # @param job Job instance with the job informations\n- #\n- def recordJob(self, job):\n- self._lJobID.append(job)\n- \n- ## Remove a job from the job table\n- #\n- # @param job: job instance to remove\n- #\n- def removeJob(self, job):\n- pass \n- \n- ## Set the jobid of a job with the id of SGE\n- #\n- # @param job job instance\n- # @param jobid integer\n- #\n- def updateJobIdInDB(self, job, jobid):\n- pass\n- \n- ## Get a job status\n- #\n- # @param job: a Job instance with the job informations\n- #\n- def getJobStatus(self, job):\n- pass\n- \n- \n- ## Change a job status\n- #\n- # @param job: a Job instance with the job informations\n- # @param status: the new status (waiting,finished,error)\n- #\n- def changeJobStatus(self, job, status):\n- pass\n- \n- ## Get the number of jobs belonging to the desired groupid with the desired status.\n- #\n- # @param groupid string a group identifier to record related job series \n- # @param status string job status (waiting, running, finished, error)\n- # @return int\n- #\n- def getCountStatus(self, groupid, status):\n- pass\n- \n- ## Clean all job from a job group\n- #\n- # @param groupid: a group identifier to record related job series\n- #\n- def cleanJobGroup(self, groupid):\n- pass \n- \n- ## Check if there is unfinished job from a job group.\n- #\n- # @param groupid string a group identifier to record related job series \n- # \n- def hasUnfinishedJob(self, groupid):\n- pass\n-\n- def _getJobIDListFromQstat(self):\n- lJobIDFromQstat = []\n- tmp = tempfile.NamedTemporaryFile(delete=False)\n- cmd ="qstat | grep %s" % self._acronym\n- process = subprocess.Popen(cmd, shell=True,stdout=tmp)\n- process.communicate()\n- tmp.close()\n- if process.returncode == 0:\n- fileName = tmp.name\n- jo'..b'ault = 0)\n- # \n- def submitJob(self, job, verbose=0, maxNbWaitingJobs=10000, checkInterval=30):\n- cmd = self._getQsubCommand(job)\n- tmp = tempfile.NamedTemporaryFile(delete=False)\n- process = subprocess.Popen(cmd, shell=True,stdout=tmp)\n- process.communicate()\n- tmp.close()\n- if process.returncode == 0:\n- fileName = tmp.name\n- jobidFileHandler = open(fileName, "r")\n- jobid = self._getJobidFromJobManager(jobidFileHandler)\n- if verbose > 0:\n- print "job \'%i %s\' submitted" % (jobid, job.jobname)\n- sys.stdout.flush()\n- job.jobid = jobid\n- #newJob= Job(job.jobid, job.jobname, job.groupid, job.queue, job.command, job.launcher, job.node, job.lResources, job.parallelEnvironment)\n- self._acronym = job.jobname.split("_")[0][:10]\n- self.recordJob(job.jobid)\n- jobidFileHandler.close()\n- os.remove(fileName)\n- return process.returncode\n-\n-\n- ## Get the list of nodes where jobs of one group were executed\n- #\n- # @param groupid string a group identifier of job series \n- # @return lNodes list of nodes names without redundancy\n- #\n- def getNodesListByGroupId(self, groupId):\n- pass\n- \n- def checkJobTable(self):\n- pass\n- \n- def close(self):\n- pass\n- \n- def _getJobidAndNbJob(self, jobid) :\n- tab = jobid.split(".")\n- jobid = tab[0]\n- tab = tab[1].split(":")\n- nbJob = tab[0]\n- return jobid, nbJob\n- \n-class JobAdaptatorSGE(JobAdaptator):\n-\n- ## Check if a job is still handled by SGE\n- #\n- # @param jobid string job identifier\n- # @param jobname string job name\n- # \n- def isJobStillHandledBySge(self, jobid, jobname):\n- isJobInQstat = False\n- tmp = tempfile.NamedTemporaryFile(delete=False)\n- cmd = "qstat"\n- process = subprocess.Popen(cmd, shell=True,stdout=tmp)\n- process.communicate()\n- tmp.close()\n- qstatFile = tmp.name\n- if process.returncode != 0:\n- msg = "ERROR while launching \'qstat\'"\n- sys.stderr.write( "%s\\n" % msg )\n- sys.exit(1)\n- qstatFileHandler = open(qstatFile, "r")\n- lLines = qstatFileHandler.readlines()\n- for line in lLines:\n- tokens = line.split()\n- if len(tokens) > 3 and tokens[0] == str(jobid) and tokens[2] == jobname[0:len(tokens[2])]:\n- isJobInQstat = True\n- break\n- qstatFileHandler.close()\n- os.remove(qstatFile)\n- return isJobInQstat\n- \n- def _getQsubCommand(self, job): \n- cmd = "echo \'%s\' | " % job.launcher\n- prg = "qsub"\n- cmd += prg\n- cmd += " -V"\n- cmd += " -N %s" % job.jobname\n- if job.queue != "":\n- cmd += " -q %s" % job.queue\n- cmd += " -cwd"\n- if job.lResources != []:\n- cmd += " -l \\""\n- cmd += " ".join(job.lResources)\n- cmd += "\\""\n- if job.parallelEnvironment != "":\n- cmd += " -pe " + job.parallelEnvironment\n- return cmd\n- \n- def _getJobidFromJobManager(self, jobidFileHandler):\n- return int(jobidFileHandler.readline().split(" ")[2])\n- \n-\n-class JobAdaptatorTorque(JobAdaptator): \n- \n- def _getQsubCommand(self, job): \n- cmd = "echo \'%s\' | " % job.launcher\n- prg = "qsub"\n- cmd += prg\n- cmd += " -V"\n- cmd += " -d %s" % os.getcwd()\n- cmd += " -N %s" % job.jobname\n- if job.queue != "":\n- cmd += " -q %s" % job.queue\n- if job.lResources != []:\n- cmd += " -l \\""\n- cmd += " ".join(job.lResources).replace("mem_free","mem")\n- cmd += "\\""\n- return cmd\n-\n- def _getJobidFromJobManager(self, jobidFileHandler):\n- return int(jobidFileHandler.readline().split(".")[0])\n' |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/OldRepetDB.py --- a/commons/core/sql/OldRepetDB.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| b |
| @@ -1,27 +0,0 @@ -import pyRepet.sql.RepetDBMySQL - - -class RepetDB ( pyRepet.sql.RepetDBMySQL.RepetDB ): - - #TODO: try - def execute( self, qry, params=None ): - if params == None: - self.cursor.execute( qry ) - else: - self.cursor.execute( qry, params ) - - - ## Record a new table in the 'info_table' table - # - # @param tablename table name - # @param info information on the origin of the table - # - def updateInfoTable( self, tablename, info ): - self.execute( """SHOW TABLES""" ) - results = self.fetchall() - if ("info_tables",) not in results: - sqlCmd = "CREATE TABLE info_tables ( name varchar(255), file varchar(255) )" - self.execute( sqlCmd ) - qryParams = "INSERT INTO info_tables VALUES (%s, %s)" - params = ( tablename, info ) - self.execute( qryParams,params ) |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/RepetJob.py --- a/commons/core/sql/RepetJob.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| b'@@ -1,252 +0,0 @@\n-# Copyright INRA (Institut National de la Recherche Agronomique)\n-# http://www.inra.fr\n-# http://urgi.versailles.inra.fr\n-#\n-# This software is governed by the CeCILL license under French law and\n-# abiding by the rules of distribution of free software. You can use, \n-# modify and/ or redistribute the software under the terms of the CeCILL\n-# license as circulated by CEA, CNRS and INRIA at the following URL\n-# "http://www.cecill.info". \n-#\n-# As a counterpart to the access to the source code and rights to copy,\n-# modify and redistribute granted by the license, users are provided only\n-# with a limited warranty and the software\'s author, the holder of the\n-# economic rights, and the successive licensors have only limited\n-# liability. \n-#\n-# In this respect, the user\'s attention is drawn to the risks associated\n-# with loading, using, modifying and/or developing or reproducing the\n-# software by the user in light of its specific status of free software,\n-# that may mean that it is complicated to manipulate, and that also\n-# therefore means that it is reserved for developers and experienced\n-# professionals having in-depth computer knowledge. Users are therefore\n-# encouraged to load and test the software\'s suitability as regards their\n-# requirements in conditions enabling the security of their systems and/or \n-# data to be ensured and, more generally, to use and operate it in the \n-# same conditions as regards security. \n-#\n-# The fact that you are presently reading this means that you have had\n-# knowledge of the CeCILL license and that you accept its terms.\n-\n-\n-import os\n-import time\n-import sys\n-from commons.core.sql.DbMySql import DbMySql\n-from commons.core.sql.TableJobAdaptatorFactory import TableJobAdaptatorFactory\n-\n-#TODO: to remove... => replace all RepetJob() by TableJobAdaptator()...\n-## Methods for Job persistence \n-#\n-class RepetJob( DbMySql ):\n- \n- \n- ## Record a job\n- #\n- # @param job Job instance with the job informations\n- #\n- def recordJob( self, job ):\n- self.removeJob( job )\n- sqlCmd = "INSERT INTO %s" % ( job.tablename )\n- sqlCmd += " VALUES ("\n- sqlCmd += " \\"%s\\"," % ( job.jobid )\n- sqlCmd += " \\"%s\\"," % ( job.jobname )\n- sqlCmd += " \\"%s\\"," % ( job.groupid )\n- sqlCmd += " \\"%s\\"," % ( job.command.replace("\\"","\\\'") )\n- sqlCmd += " \\"%s\\"," % ( job.launcher )\n- sqlCmd += " \\"%s\\"," % ( job.queue )\n- sqlCmd += " \\"waiting\\","\n- sqlCmd += " \\"%s\\"," % ( time.strftime( "%Y-%m-%d %H:%M:%S" ) )\n- sqlCmd += " \\"?\\" );"\n- self.execute( sqlCmd )\n- \n- \n- ## Remove a job from the job table\n- #\n- # @param job: job instance to remove\n- #\n- def removeJob( self, job ):\n- qry = "DELETE FROM %s" % ( job.tablename )\n- qry += " WHERE groupid=\'%s\'" % ( job.groupid )\n- qry += " AND jobname=\'%s\'" % ( job.jobname )\n- qry += " AND queue=\'%s\';" % ( job.queue )\n- self.execute( qry )\n- \n- \n- ## Set the jobid of a job with the id of SGE\n- #\n- # @param job job instance\n- # @param jobid integer\n- #\n- def setJobIdFromSge( self, job, jobid ):\n- qry = "UPDATE %s" % ( job.tablename )\n- qry += " SET jobid=\'%i\'" % ( int(jobid) )\n- qry += " WHERE jobname=\'%s\'" % ( job.jobname )\n- qry += " AND groupid=\'%s\'" % ( job.groupid )\n- qry += " AND queue=\'%s\';" % ( job.queue )\n- self.execute( qry )\n- \n- \n- ## Get a job status\n- #\n- # @param job: a Job instance with the job informations\n- #\n- def getJobStatus( self, job ):\n- if job.jobid != 0 and job.jobname == "":\n- job.jobname = job.jobid\n- job.jobid = 0\n- qry = "SELECT status FROM %s" % ( job.tablename )\n- qry += " WHERE groupid=\'%s\'" % ( job.groupid )\n- qry += " AND jobname=\'%s\'" % ( job.jobname )\n- qry += " '..b' table name to record the jobs\n- # @param groupid string a group identifier to record related job series \n- # \n- def hasUnfinishedJob( self, tablename, groupid ):\n- if not self.doesTableExist( tablename ):\n- return False\n- qry = "SELECT * FROM %s" % ( tablename )\n- qry += " WHERE groupid=\'%s\'" % ( groupid )\n- qry += " and status!=\'finished\';" \n- self.execute( qry )\n- res = self.fetchall()\n- if len(res) == 0:\n- return False\n- return True\n- \n- \n- ## Check if a job is still handled by SGE\n- #\n- # @param jobid string job identifier\n- # @param jobname string job name\n- # \n- def isJobStillHandledBySge( self, jobid, jobname ):\n- isJobInQstat = False\n- qstatFile = "qstat_stdout"\n- cmd = "qstat > %s" % ( qstatFile )\n- returnStatus = os.system( cmd )\n- if returnStatus != 0:\n- msg = "ERROR while launching \'qstat\'"\n- sys.stderr.write( "%s\\n" % msg )\n- sys.exit(1)\n- qstatFileHandler = open( qstatFile, "r" )\n- lLines = qstatFileHandler.readlines()\n- for line in lLines:\n- tokens = line.split()\n- if len(tokens) > 3 and tokens[0] == str(jobid) and tokens[2] == jobname[0:len(tokens[2])]:\n- isJobInQstat = True\n- break\n- qstatFileHandler.close()\n- os.remove( qstatFile )\n- return isJobInQstat\n- \n- \n- ## Wait job finished status from a job group.\n- # Job are re-launched if error (max. 3 times)\n- #\n- # @param tableName string table name to record the jobs\n- # @param groupid string a group identifier to record related job series\n- # @param checkInterval integer time laps in seconds between two checks (default = 5)\n- # @param maxRelaunch integer max nb of times a job in error is relaunch before exiting (default = 3)\n- # @param exitIfTooManyErrors boolean exit if a job is still in error above maxRelaunch (default = True)\n- # @param timeOutPerJob integer max nb of seconds after which one tests if a job is still in SGE or not (default = 60*60=1h)\n- #\n- def waitJobGroup(self, tableName, groupid, checkInterval=5, maxRelaunch=3, exitIfTooManyErrors=True, timeOutPerJob=60*60):\n- iTJA = TableJobAdaptatorFactory.createInstance(self, tableName)\n- iTJA.waitJobGroup(groupid, checkInterval, maxRelaunch, exitIfTooManyErrors, timeOutPerJob)\n- \n- ## Submit a job to a queue and record it in job table.\n- #\n- # @param job a job instance\n- # @param maxNbWaitingJobs integer max nb of waiting jobs before submitting a new one (default = 10000)\n- # @param checkInterval integer time laps in seconds between two checks (default = 30)\n- # @param verbose integer (default = 0)\n- # \n- def submitJob( self, job, verbose=0, maxNbWaitingJobs=10000, checkInterval=30 ):\n- iTJA = TableJobAdaptatorFactory.createInstance(self, job.tablename)\n- return iTJA.submitJob(job, verbose, maxNbWaitingJobs, checkInterval)\n- \n- \n- ## Get the list of nodes where jobs of one group were executed\n- #\n- # @param tablename string table name where jobs are recored \n- # @param groupid string a group identifier of job series \n- # @return lNodes list of nodes names\n- #\n- def getNodesListByGroupId( self, tableName, groupId ):\n- qry = "SELECT node FROM %s" % tableName\n- qry += " WHERE groupid=\'%s\'" % groupId\n- self.execute( qry )\n- res = self.fetchall()\n- lNodes = []\n- for resTuple in res:\n- lNodes.append(resTuple[0])\n- return lNodes\n- \n- def getDbName(self):\n- return "DbMySql"\n- \n- def _getJobidAndNbJob(self, jobid) :\n- tab = []\n- tab = jobid.split(".")\n- jobid = tab[0]\n- tab = tab[1].split(":")\n- nbJob = tab[0]\n- return jobid, nbJob\n' |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/TableAdaptator.py --- a/commons/core/sql/TableAdaptator.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| b |
| @@ -1,128 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - - -## Abstract class, Ancestor of Table*Adaptator -# -class TableAdaptator( object ): - - ## Constructor - # - # @param iDb DbMySql instance - # @param table str table name - # - def __init__( self, iDb = None, table = "" ): - self._iDb = iDb - self._table = table - - ## Set connector to database - # - # @param iDb database instance - # - def setDbConnector( self, iDb ): - self._iDb = iDb - - ## Set table - # - # @param table string table name - # - def setTable( self, table ): - self._table = table - - ## Return the table name - # - def getTable( self ): - return self._table - - ## Return the number of rows in the table - # - def getSize( self ): - return self._iDb.getSize( self._table ) - - ## Test if table is empty - # - def isEmpty( self ): - return self._iDb.isEmpty( self._table ) - - ## Insert an instance of Map or Set or Match or Path or Seq instances - # - # @param obj a Map or Set or Match or Path or Seq instance - # @param delayed boolean - # - def insert(self, obj, delayed = False): - if obj.isEmpty(): - return - self._escapeAntislash(obj) - sql_cmd = self._genSqlCmdForInsert(obj, delayed) - self._iDb.execute(sql_cmd) - - ## Insert a list of Map or Set or Match or Path instances - # - # @param l a list of object instances - # @param delayed boolean - # - def insertList(self, l, delayed = False): - for i in l: - self.insert(i, delayed) - - ## Give the data contained in the table as a list of coord object instances - # - # @return lObject list of coord object instances - # - def getListOfAllCoordObject( self ): - sqlCmd = "SELECT * FROM %s" % ( self._table ) - lObjs = self._iDb.getObjectListWithSQLCmd( sqlCmd, self._getInstanceToAdapt ) - return lObjs - - ## Generate sql command for GetListOverlappingCoord method - # - # @param obj Map, Set or Match instance - # @param delayed boolean - # @return sqlCmd string generated sql command - # - def _genSqlCmdForInsert(self, obj, delayed): - sqlCmd = 'INSERT ' - if delayed : - sqlCmd += ' DELAYED ' - type2Insert, attr2Insert = self._getTypeAndAttr2Insert(obj) - sqlCmd += 'INTO %s VALUES (' % (self._table) - sqlCmd += ",".join(type2Insert) - sqlCmd += ")" - sqlCmd = sqlCmd % attr2Insert - return sqlCmd - - def _getTypeAndAttr2Insert(self, obj): - pass - - def _getInstanceToAdapt(self): - pass - - def _escapeAntislash(self, obj): - pass |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/TableBinPathAdaptator.py --- a/commons/core/sql/TableBinPathAdaptator.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| b'@@ -1,257 +0,0 @@\n-# Copyright INRA (Institut National de la Recherche Agronomique)\n-# http://www.inra.fr\n-# http://urgi.versailles.inra.fr\n-#\n-# This software is governed by the CeCILL license under French law and\n-# abiding by the rules of distribution of free software. You can use, \n-# modify and/ or redistribute the software under the terms of the CeCILL\n-# license as circulated by CEA, CNRS and INRIA at the following URL\n-# "http://www.cecill.info". \n-#\n-# As a counterpart to the access to the source code and rights to copy,\n-# modify and redistribute granted by the license, users are provided only\n-# with a limited warranty and the software\'s author, the holder of the\n-# economic rights, and the successive licensors have only limited\n-# liability. \n-#\n-# In this respect, the user\'s attention is drawn to the risks associated\n-# with loading, using, modifying and/or developing or reproducing the\n-# software by the user in light of its specific status of free software,\n-# that may mean that it is complicated to manipulate, and that also\n-# therefore means that it is reserved for developers and experienced\n-# professionals having in-depth computer knowledge. Users are therefore\n-# encouraged to load and test the software\'s suitability as regards their\n-# requirements in conditions enabling the security of their systems and/or \n-# data to be ensured and, more generally, to use and operate it in the \n-# same conditions as regards security. \n-#\n-# The fact that you are presently reading this means that you have had\n-# knowledge of the CeCILL license and that you accept its terms.\n-\n-from commons.core.coord.Range import getIdx\n-from commons.core.sql.TablePathAdaptator import TablePathAdaptator\n-from commons.core.coord.PathUtils import PathUtils\n-\n-## Bin Adaptator for a path table.\n-#\n-class TableBinPathAdaptator(TablePathAdaptator):\n-\n- \n- ## Constructor\n- #\n- # @param db db instance\n- # @param tableName string table name (default = "")\n- #\n- def __init__(self, db, tableName = ""):\n- TablePathAdaptator.__init__(self, db, tableName)\n- self._table_idx = "%s_idx" % (self._table)\n- \n- ## Insert a path instance\n- #\n- # @param path a path instance\n- # @param delayed boolean indicating if the insert must be delayed (default = false) \n- # \n- def insert( self, path, delayed = False ):\n- TablePathAdaptator.insert(self, path, delayed)\n- self._escapeAntislash(path)\n- idx = path.range_query.findIdx()\n- max = path.range_query.getMax()\n- min = path.range_query.getMin()\n- strand = path.range_query.isOnDirectStrand()\n- if delayed:\n- sql_cmd = \'INSERT DELAYED INTO %s VALUES (%d,%d,"%s",%d,%d,%d)\'\\\n- % (self._table_idx,\\\n- path.id,\\\n- idx,\\\n- path.range_query.seqname,\\\n- min,\\\n- max,\\\n- strand)\n- else:\n- sql_cmd = \'INSERT INTO %s VALUES (%d,%d,"%s",%d,%d,%d)\'\\\n- % (self._table_idx,\\\n- path.id,\\\n- idx,\\\n- path.range_query.seqname,\\\n- min,\\\n- max,\\\n- strand)\n- \n- self._iDb.execute(sql_cmd)\n- \n- ## Return a path instances list included in a given region using the bin scheme\n- #\n- # @param contig string contig name\n- # @param start integer start coordinate\n- # @param end integer end coordinate\n- # @return lOutPath a path instances list\n- #\n- def getPathListIncludedInQueryCoord(self, contig, start, end):\n- min_coord = min(start, end)\n- max_coord = max(start, end)\n- lpath = self.getChainListOverlappingQueryCoord(contig, start, end)\n- lOutPath = []\n- for i in lpath:\n- if i.range_query.getMin() > min_coord and \\\n- i.range_query.getMax() < max_'..b' \n- sql_cmd += ") and min<=%d and max>=%d;" % (max_coord, min_coord)\n-\n- \n- self._iDb.execute(sql_cmd)\n- res = self._iDb.fetchall()\n- lnum = []\n- for i in res:\n- lnum.append( int(i[0]) )\n- lpath = self.getPathListFromIdList(lnum)\n- return lpath\n-\n- ## Delete path corresponding to a given identifier number\n- #\n- # @param num integer identifier number\n- #\n- def deleteFromId(self, num):\n- TablePathAdaptator.deleteFromId(self, num)\n- sqlCmd=\'delete from %s where path=%d;\' % (self._table_idx, num)\n- self._iDb.execute(sqlCmd)\n- \n- ## Delete path corresponding to a given list of identifier number\n- #\n- # @param lNum list list of integer identifier number\n- #\n- def deleteFromIdList(self, lNum):\n- if lNum == []:\n- return\n- TablePathAdaptator.deleteFromIdList(self, lNum)\n- sqlCmd = \'delete from %s where path=%d\' % (self._table_idx, lNum[0])\n- for i in lNum[1:]:\n- sqlCmd += " or path=%d" % (i)\n- sqlCmd += ";"\n- self._iDb.execute(sqlCmd)\n- \n- ## Join two path by changing id number of id1 and id2 path to the least of id1 and id2\n- #\n- # @param id1 integer id path number\n- # @param id2 integer id path number\n- # @return newId integer minimum of id1 id2\n- # @note this method modify the ID even if this one not existing in the path table \n- # \n- def joinTwoPaths(self, id1, id2):\n- TablePathAdaptator.joinTwoPaths(self, id1, id2)\n- if id1 < id2:\n- newId = id1\n- oldId = id2\n- else:\n- newId = id2\n- oldId = id1\n- sqlCmd = \'UPDATE %s SET path=%d WHERE path=%d\' % (self._table_idx, newId, oldId)\n- self._iDb.execute(sqlCmd)\n- return newId\n- \n- ## Get a new id number\n- #\n- # @return newId integer max Id in path table + 1\n- #\n- def getNewId(self):\n- sqlCmd = \'select max(path) from %s;\' % (self._table_idx)\n- self._iDb.execute(sqlCmd)\n- maxId = self._iDb.fetchall()[0][0]\n- if maxId == None:\n- maxId = 0\n- newId = int(maxId) + 1\n- return newId\n- \n- ## Give a list of Set instances included in a given region\n- #\n- # @param query string query name\n- # @param start integer start coordinate\n- # @param end integer end coordinate\n- # @return lSet list of Set instances\n- #\n- def getSetListIncludedInQueryCoord(self, query, start, end):\n- lPath=self.getPathListIncludedInQueryCoord(query, start, end)\n- lSet = PathUtils.getSetListFromQueries(lPath) \n- return lSet\n- \n- ## Give a list of Set instances overlapping a given region\n- #\n- # @param query string query name\n- # @param start integer start coordinate\n- # @param end integer end coordinate\n- # @return lSet list of Set instances\n- #\n- def getSetListOverlappingQueryCoord(self, query, start, end):\n- lPath = self.getPathListOverlappingQueryCoord(query, start, end)\n- lSet = PathUtils.getSetListFromQueries(lPath)\n- return lSet\n- \n- ## Give a list of identifiers contained in the table\n- #\n- # @return lId integer list\n- #\n- def getIdList(self):\n- sqlCmd = "SELECT DISTINCT path from %s;" % (self._table_idx)\n- lId = self._iDb.getIntegerListWithSQLCmd( sqlCmd )\n- return lId\n- \n- ## Give a list of the distinct query names present in the table\n- #\n- # @return lDistinctQueryNames string list\n- #\n- def getQueryList(self):\n- lDistinctQueryNames = self._getDistinctTypeNamesList("query")\n- return lDistinctQueryNames\n- \n- def _getDistinctTypeNamesList( self, type ):\n- sqlCmd = "SELECT DISTINCT contig FROM %s" % ( self._table_idx )\n- lDistinctTypeNames = self._iDb.getStringListWithSQLCmd(sqlCmd)\n- return lDistinctTypeNames\n\\ No newline at end of file\n' |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/TableBinSetAdaptator.py --- a/commons/core/sql/TableBinSetAdaptator.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| b'@@ -1,265 +0,0 @@\n-# Copyright INRA (Institut National de la Recherche Agronomique)\n-# http://www.inra.fr\n-# http://urgi.versailles.inra.fr\n-#\n-# This software is governed by the CeCILL license under French law and\n-# abiding by the rules of distribution of free software. You can use, \n-# modify and/ or redistribute the software under the terms of the CeCILL\n-# license as circulated by CEA, CNRS and INRIA at the following URL\n-# "http://www.cecill.info". \n-#\n-# As a counterpart to the access to the source code and rights to copy,\n-# modify and redistribute granted by the license, users are provided only\n-# with a limited warranty and the software\'s author, the holder of the\n-# economic rights, and the successive licensors have only limited\n-# liability. \n-#\n-# In this respect, the user\'s attention is drawn to the risks associated\n-# with loading, using, modifying and/or developing or reproducing the\n-# software by the user in light of its specific status of free software,\n-# that may mean that it is complicated to manipulate, and that also\n-# therefore means that it is reserved for developers and experienced\n-# professionals having in-depth computer knowledge. Users are therefore\n-# encouraged to load and test the software\'s suitability as regards their\n-# requirements in conditions enabling the security of their systems and/or \n-# data to be ensured and, more generally, to use and operate it in the \n-# same conditions as regards security. \n-#\n-# The fact that you are presently reading this means that you have had\n-# knowledge of the CeCILL license and that you accept its terms.\n-\n-from commons.core.sql.TableSetAdaptator import TableSetAdaptator\n-from commons.core.coord.SetUtils import SetUtils\n-\n-## Adaptator for Set tables with bin indexes\n-#\n-class TableBinSetAdaptator(TableSetAdaptator):\n- \n- ## constructor\n- #\n- # @param iDb DbMySql instance instance of DbMySql\n- # @param tableName string table name (default = "")\n- #\n- def __init__(self, iDb, tableName = ""):\n- TableSetAdaptator.__init__(self, iDb, tableName)\n- self._table_idx = "%s_idx" % (self._table)\n- \n- ## Insert a set instance in a set bin table\n- # \n- # @param iSet set instance an instance of set object\n- # @param delayed boolean an insert delayed or not\n- #\n- def insASetInSetAndBinTable(self, iSet, delayed = False):\n- self.insert(iSet, delayed)\n- iSet.seqname = iSet.seqname.replace("\\\\", "\\\\\\\\")\n- iSet.name = iSet.name.replace("\\\\", "\\\\\\\\")\n- bin = iSet.getBin()\n- max = iSet.getMax()\n- min = iSet.getMin()\n- strand = iSet.isOnDirectStrand()\n- sql_prefix = \'\'\n- if delayed:\n- sql_prefix = \'INSERT DELAYED INTO \'\n- else:\n- sql_prefix = \'INSERT INTO \'\n- sql_cmd = sql_prefix + \'%s VALUES (%d,%f,"%s",%d,%d,%d)\'\\\n- %(self._table_idx,\\\n- iSet.id,\\\n- bin,\\\n- iSet.seqname,\\\n- min,\\\n- max,\\\n- strand)\n- self._iDb.execute(sql_cmd)\n-\n- ## Delete set corresponding to a given identifier number in set and bin set table\n- # @param id integer identifier number\n- # @note old name was delSet_from_num\n- #\n- def deleteFromIdFromSetAndBinTable(self, id):\n- self.deleteFromId(id)\n- sql_cmd = \'delete from %s where path=%d\' % (self._table_idx, id)\n- self._iDb.execute(sql_cmd)\n-\n- ## Delete path corresponding to a given list of identifier number\n- #\n- # @param lId integer list list of identifier number\n- # @note old name was delSet_from_listnum\n- #\n- def deleteFromListIdFromSetAndBinTable(self, lId):\n- if lId != []:\n- self.deleteFromIdList(lId)\n- sql_cmd = \'delete from %s where path=%d\' % (self._table_idx, lId[0])\n- for i in lId[1:]:\n- sql_cmd += " or path=%d" % (i)\n- self.'..b"has been changed : I added the two first lines\n- #\n- def getSetListStrictlyIncludedInQueryCoord(self, contig, start, end):\n- min_coord = min(start,end)\n- max_coord = max(start,end)\n- lSet = self.getSetListFromQueryCoord(contig, start, end) \n- lSetStrictlyIncluded = []\n- for iSet in lSet:\n- if iSet.getMin() > min_coord and \\\n- iSet.getMax() < max_coord:\n- lSetStrictlyIncluded.append(iSet)\n- \n- return lSetStrictlyIncluded\n- \n- ## Get a list of the identifier Id contained in the table bin\n- #\n- # @return lId list of int list of identifier\n- # @note old name was getSet_num\n- #\n- def getIdList(self):\n- sql_cmd = 'select distinct path from %s;' % (self._table_idx)\n- self._iDb.execute(sql_cmd)\n- res = self._iDb.fetchall()\n- lId = []\n- for t in res:\n- lId.append(int(t[0]))\n- return lId\n- \n- ## Get a list of the query sequence name contained in the table bin\n- #\n- # @return lSeqName list of string list of query sequence name\n- # @note old name was getContig_name\n- #\n- def getSeqNameList(self):\n- sql_cmd = 'select distinct contig from %s;' % (self._table_idx)\n- self._iDb.execute(sql_cmd)\n- res = self._iDb.fetchall()\n- lSeqName = []\n- for t in res:\n- lSeqName.append(t[0])\n- return lSeqName\n- \n- ## Insert a Set list with the same new identifier in the table bin and set\n- #\n- # @note old name was insAddSetList\n- #\n- def insertListInSetAndBinTable(self, lSets, delayed = False):\n- id = self.getNewId()\n- SetUtils.changeIdInList( lSets, id )\n- for iSet in lSets:\n- self.insASetInSetAndBinTable(iSet, delayed)\n- \n- ## Insert a set list instances In table Bin and Set and merge all overlapping sets\n- #\n- # @param lSets reference seq name\n- # @note old name was insMergeSetList\n- # \n- def insertListInSetAndBinTableAndMergeAllSets(self, lSets):\n- min, max = SetUtils.getListBoundaries(lSets)\n- oldLSet = self.getSetListFromQueryCoord(lSets[0].seqname, min, max)\n- oldQueryhash = SetUtils.getDictOfListsWithIdAsKey(oldLSet)\n- qhash = SetUtils.getDictOfListsWithIdAsKey(lSets)\n- for lNewSetById in qhash.values():\n- found = False\n- for currentId, oldLsetById in oldQueryhash.items():\n- if SetUtils.areSetsOverlappingBetweenLists(lNewSetById, oldLsetById):\n- oldLsetById.extend(lNewSetById)\n- oldLsetById = SetUtils.mergeSetsInList(oldLsetById)\n- self.deleteFromIdFromSetAndBinTable(currentId)\n- found = True\n- if not found:\n- self.insertListInSetAndBinTable(lNewSetById)\n- else:\n- id = self.getNewId()\n- SetUtils.changeIdInList(oldLsetById, id)\n- self.insertListInSetAndBinTable(oldLsetById)\n- \n- ## Insert a set list instances In table Bin and Set after removing all overlaps between database and lSets\n- #\n- # @param lSets reference seq name\n- # @note old name was insDiffSetList\n- # \n- def insertListInSetAndBinTableAndRemoveOverlaps(self, lSets):\n- min, max = SetUtils.getListBoundaries(lSets)\n- oldLSet = self.getSetListFromQueryCoord(lSets[0].seqname, min, max)\n- oldQueryHash = SetUtils.getDictOfListsWithIdAsKey(oldLSet)\n- newQueryHash = SetUtils.getDictOfListsWithIdAsKey(lSets)\n- for lNewSetById in newQueryHash.values():\n- for lOldSetById in oldQueryHash.values():\n- if SetUtils.areSetsOverlappingBetweenLists(lNewSetById, lOldSetById):\n- lNewSetById = SetUtils.getListOfSetWithoutOverlappingBetweenTwoListOfSet(lOldSetById, lNewSetById)\n- self.insertListInSetAndBinTable(lNewSetById)\n" |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/TableJobAdaptator.py --- a/commons/core/sql/TableJobAdaptator.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| b'@@ -1,405 +0,0 @@\n-# Copyright INRA (Institut National de la Recherche Agronomique)\n-# http://www.inra.fr\n-# http://urgi.versailles.inra.fr\n-#\n-# This software is governed by the CeCILL license under French law and\n-# abiding by the rules of distribution of free software. You can use, \n-# modify and/ or redistribute the software under the terms of the CeCILL\n-# license as circulated by CEA, CNRS and INRIA at the following URL\n-# "http://www.cecill.info". \n-#\n-# As a counterpart to the access to the source code and rights to copy,\n-# modify and redistribute granted by the license, users are provided only\n-# with a limited warranty and the software\'s author, the holder of the\n-# economic rights, and the successive licensors have only limited\n-# liability. \n-#\n-# In this respect, the user\'s attention is drawn to the risks associated\n-# with loading, using, modifying and/or developing or reproducing the\n-# software by the user in light of its specific status of free software,\n-# that may mean that it is complicated to manipulate, and that also\n-# therefore means that it is reserved for developers and experienced\n-# professionals having in-depth computer knowledge. Users are therefore\n-# encouraged to load and test the software\'s suitability as regards their\n-# requirements in conditions enabling the security of their systems and/or \n-# data to be ensured and, more generally, to use and operate it in the \n-# same conditions as regards security. \n-#\n-# The fact that you are presently reading this means that you have had\n-# knowledge of the CeCILL license and that you accept its terms.\n-\n-\n-import os\n-import time\n-import datetime\n-import sys\n-from commons.core.sql.Job import Job \n-from commons.core.sql.TableAdaptator import TableAdaptator\n-\n-## Methods for Job persistence \n-#\n-class TableJobAdaptator(TableAdaptator):\n- \n- ## Record a job\n- #\n- # @param job Job instance with the job informations\n- #\n- def recordJob(self, job):\n- self.removeJob(job)\n- sqlCmd = "INSERT INTO %s" % self._table\n- sqlCmd += " VALUES ("\n- sqlCmd += " \\"%s\\"," % job.jobid\n- sqlCmd += " \\"%s\\"," % job.jobname\n- sqlCmd += " \\"%s\\"," % job.groupid\n- sqlCmd += " \\"%s\\"," % job.launcher\n- sqlCmd += " \\"%s\\"," % job.queue\n- sqlCmd += " \\"%s\\"," % job.lResources\n- sqlCmd += " \\"waiting\\","\n- sqlCmd += " \\"%s\\"," % time.strftime("%Y-%m-%d %H:%M:%S")\n- sqlCmd += " \\"?\\" );"\n- self._iDb.execute(sqlCmd)\n- \n- \n- ## Remove a job from the job table\n- #\n- # @param job: job instance to remove\n- #\n- def removeJob(self, job):\n- qry = "DELETE FROM %s" % self._table\n- qry += " WHERE groupid=\'%s\'" % job.groupid\n- qry += " AND jobname=\'%s\'" % job.jobname\n- qry += " AND launcher=\'%s\';" % job.launcher\n- self._iDb.execute(qry)\n- \n- \n- ## Set the jobid of a job with the id of SGE\n- #\n- # @param job job instance\n- # @param jobid integer\n- #\n- def updateJobIdInDB(self, job, jobid):\n- #TODO: check if only one job will be updated\n- qry = "UPDATE %s" % self._table\n- qry += " SET jobid=\'%i\'" % int(jobid)\n- qry += " WHERE jobname=\'%s\'" % job.jobname\n- qry += " AND groupid=\'%s\'" % job.groupid\n- qry += " AND launcher=\'%s\';" % job.launcher\n- self._iDb.execute(qry)\n- \n- \n- ## Get a job status\n- #\n- # @param job: a Job instance with the job informations\n- #\n- def getJobStatus(self, job):\n- if job.jobid != 0 and job.jobname == "":\n- job.jobname = job.jobid\n- job.jobid = 0\n- qry = "SELECT status FROM %s" % self._table\n- qry += " WHERE groupid=\'%s\'" % job.groupid\n- qry += " AND jobname=\'%s\'" % job.jobname\n- qry += " AND launcher=\'%s\';" % job.launcher\n- self._iDb.execute(qry)\n- res = self._iDb.fetchall()\n- if len(re'..b'outside the interval: go to next interval (time out) \n- if delta.seconds >= (nbTimeOuts+1) * timeOutPerJob:\n- nbTimeOuts += 1\n- # Job with \'running\' status should be in qstat. Because status in DB is set at \'running\' by the job launched.\n- if not self.isJobStillHandledBySge(jobid, jobname):\n- # But if not, let time for the status update (in DB), if the job finished between the query execution and now.\n- time.sleep( 5 )\n- # If no update at \'finished\', exit\n- #TODO: check status in DB\n- if not self.isJobStillHandledBySge(jobid, jobname):\n- msg = "ERROR: job \'%s\', supposedly still running, is not handled by SGE anymore" % ( jobid )\n- msg += "\\nit was launched the %s (> %.2f hours ago)" % ( dateTimeOldestJob, timeOutPerJob/3600.0 )\n- msg += "\\nthis problem can be due to:"\n- msg += "\\n* memory shortage, in that case, decrease the size of your jobs;"\n- msg += "\\n* timeout, in that case, decrease the size of your jobs;"\n- msg += "\\n* node failure or database error, in that case, launch the program again or ask your system administrator."\n- sys.stderr.write("%s\\n" % msg)\n- sys.stderr.flush()\n- self.cleanJobGroup(groupid)\n- sys.exit(1)\n- return nbTimeOuts\n- \n- ## Check if a job is still handled by SGE\n- #\n- # @param jobid string job identifier\n- # @param jobname string job name\n- # \n- def isJobStillHandledBySge(self, jobid, jobname):\n- isJobInQstat = False\n- qstatFile = "qstat_stdout"\n- cmd = "qstat > %s" % qstatFile\n- returnStatus = os.system(cmd)\n- if returnStatus != 0:\n- msg = "ERROR while launching \'qstat\'"\n- sys.stderr.write( "%s\\n" % msg )\n- sys.exit(1)\n- qstatFileHandler = open(qstatFile, "r")\n- lLines = qstatFileHandler.readlines()\n- for line in lLines:\n- tokens = line.split()\n- if len(tokens) > 3 and tokens[0] == str(jobid) and tokens[2] == jobname[0:len(tokens[2])]:\n- isJobInQstat = True\n- break\n- qstatFileHandler.close()\n- os.remove(qstatFile)\n- return isJobInQstat\n- \n- def _getQsubCommand(self, job): \n- cmd = "echo \'%s\' | " % job.launcher\n- prg = "qsub"\n- cmd += prg\n- cmd += " -V"\n- cmd += " -N %s" % job.jobname\n- if job.queue != "":\n- cmd += " -q %s" % job.queue\n- cmd += " -cwd"\n- if job.lResources != []:\n- cmd += " -l \\""\n- cmd += " ".join(job.lResources)\n- cmd += "\\""\n- if job.parallelEnvironment != "":\n- cmd += " -pe " + job.parallelEnvironment\n- cmd += " > jobid.stdout"\n- return cmd\n- \n- def _getJobidFromJobManager(self, jobidFileHandler):\n- return int(jobidFileHandler.readline().split(" ")[2])\n- \n-\n-class TableJobAdaptatorTorque(TableJobAdaptator): \n- \n- def _checkIfJobsTableAndJobsManagerInfoAreConsistent(self, nbTimeOuts, timeOutPerJob, groupid):\n- return nbTimeOuts\n- \n- def _getQsubCommand(self, job): \n- cmd = "echo \'%s\' | " % job.launcher\n- prg = "qsub"\n- cmd += prg\n- cmd += " -V"\n- cmd += " -d %s" % os.getcwd()\n- cmd += " -N %s" % job.jobname\n- if job.queue != "":\n- cmd += " -q %s" % job.queue\n- if job.lResources != []:\n- cmd += " -l \\""\n- cmd += " ".join(job.lResources).replace("mem_free","mem")\n- cmd += "\\""\n- cmd += " > jobid.stdout"\n- return cmd\n-\n- def _getJobidFromJobManager(self, jobidFileHandler):\n- return int(jobidFileHandler.readline().split(".")[0])\n' |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/TableJobAdaptatorFactory.py --- a/commons/core/sql/TableJobAdaptatorFactory.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| @@ -1,66 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - -import os -import sys -from commons.core.sql.TableJobAdaptator import TableJobAdaptatorSGE -from commons.core.sql.TableJobAdaptator import TableJobAdaptatorTorque -from commons.core.sql.JobAdaptator import JobAdaptatorSGE -from commons.core.sql.JobAdaptator import JobAdaptatorTorque - -class TableJobAdaptatorFactory(object): - - def createInstance(iDb, jobTableName): - if os.environ["REPET_JOB_MANAGER"].lower() == "sge": - iTJA = TableJobAdaptatorSGE(iDb, jobTableName) - elif os.environ["REPET_JOB_MANAGER"].lower() == "torque": - iTJA = TableJobAdaptatorTorque(iDb, jobTableName) - else: - print "ERROR: unknown jobs manager : $REPET_JOB_MANAGER = %s." % os.environ["REPET_JOB_MANAGER"] - sys.exit(1) - - return iTJA - - createInstance = staticmethod(createInstance) - - def createJobInstance(): - if os.environ["REPET_JOB_MANAGER"].lower() == "sge": - iJA = JobAdaptatorSGE() - elif os.environ["REPET_JOB_MANAGER"].lower() == "torque": - iJA = JobAdaptatorTorque() - else: - print "ERROR: unknown jobs manager : $REPET_JOB_MANAGER = %s." % os.environ["REPET_JOB_MANAGER"] - sys.exit(1) - - return iJA - - - createJobInstance = staticmethod(createJobInstance) - \ No newline at end of file |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/TableMapAdaptator.py --- a/commons/core/sql/TableMapAdaptator.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| @@ -1,193 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - - -import sys -from commons.core.sql.TableAdaptator import TableAdaptator -from commons.core.sql.ITableMapAdaptator import ITableMapAdaptator -from commons.core.coord.Map import Map -from commons.core.coord.MapUtils import MapUtils - - -## Adaptator for Map table -# -class TableMapAdaptator( TableAdaptator, ITableMapAdaptator ): - - ## Give a list of Map instances having a given seq name - # - # @param seqName string seq name - # @return lMap list of instances - # - def getListFromSeqName( self, seqName ): - sqlCmd = "SELECT * FROM %s" % (self._table) - colum2Get, type2Get, attr2Get = self._getTypeColumAttr2Get(seqName) - sqlCmd += " WHERE " + colum2Get - sqlCmd += " = " - sqlCmd = sqlCmd + type2Get - sqlCmd = sqlCmd % "'" + attr2Get + "'" - return self._iDb.getObjectListWithSQLCmd( sqlCmd, self._getInstanceToAdapt ) - - ## Give a list of Map instances overlapping a given region - # - # @param query string query name - # @param start integer start coordinate - # @param end integer end coordinate - # @return list map instances - # - def getListOverlappingCoord(self, query, start, end): - sqlCmd = 'select * from %s where chr="%s" and ((start between least(%d,%d) and greatest(%d,%d) or end between least(%d,%d) and greatest(%d,%d)) or (least(start,end)<=least(%d,%d) and greatest(start,end)>=greatest(%d,%d))) ;' % (self._table, query, start, end, start, end, start, end, start, end, start, end, start, end) - return self._iDb.getObjectListWithSQLCmd( sqlCmd, self._getInstanceToAdapt ) - - ## Give a list of Map instances having a given sequence name - # - # @param seqName string sequence name - # @return lMap list of instances - # - def getMapListFromSeqName(self, seqName): - lMap = self.getListFromSeqName( seqName ) - return lMap - -#TODO: Check getListFromSeqName method: uses name instead of seqname -# ## Give a list of Map instances having a given sequence name from list -# # -# # @param lSeqName string sequence name list -# # @return lMap list of instances -# # -# def getMapListFromSeqNameList(self, lSeqName): -# lMap = [] -# [lMap.extend(self.getListFromSeqName(seqName)) for seqName in lSeqName] -# return lMap - - ## Give a list of Map instances having a given chromosome - # - # @param chr string chromosome - # @return lMap list of instances - # - def getMapListFromChr(self, chr): - sqlCmd = "SELECT * FROM %s WHERE chr='%s'" % (self._table, chr) - lMap = self._iDb.getObjectListWithSQLCmd( sqlCmd, self._getInstanceToAdapt ) - return lMap - - ## Give a list of the distinct seqName/chr present in the table - # - # @return lDistinctContigNames string list - # - def getSeqNameList(self): - sqlCmd = "SELECT DISTINCT chr FROM %s" % ( self._table ) - lDistinctContigNames = self._iDb.getStringListWithSQLCmd(sqlCmd) - return lDistinctContigNames - - ## Return a list of Set instances from a given sequence name - # - # @param seqName string sequence name - # @return lSets list of Set instances - # - def getSetListFromSeqName( self, seqName ): - lMaps = self.getListFromSeqName( seqName ) - lSets = MapUtils.mapList2SetList( lMaps ) - return lSets - - ## Give a map instances list overlapping a given region - # - # @param seqName string seq name - # @param start integer start coordinate - # @param end integer end coordinate - # @return lMap list of map instances - # - def getMapListOverlappingCoord(self, seqName, start, end): - lMap = self.getListOverlappingCoord(seqName, start, end) - return lMap - - ## Return a list of Set instances overlapping a given sequence - # - # @param seqName string sequence name - # @param start integer start coordinate - # @param end integer end coordinate - # @return lSet list of Set instances - # - def getSetListOverlappingCoord( self, seqName, start, end ): - lMaps = self.getListOverlappingCoord( seqName, start, end ) - lSets = MapUtils.mapList2SetList( lMaps ) - return lSets - - ## Give a dictionary which keys are Map names and values the corresponding Map instances - # - # @return dName2Maps dict which keys are Map names and values the corresponding Map instances - # - def getDictPerName( self ): - dName2Maps = {} - lMaps = self.getListOfAllMaps() - for iMap in lMaps: - if dName2Maps.has_key( iMap.name ): - if iMap == dName2Maps[ iMap.name ]: - continue - else: - msg = "ERROR: in table '%s' two different Map instances have the same name '%s'" % ( self._table, iMap.name ) - sys.stderr.write( "%s\n" % ( msg ) ) - sys.exit(1) - dName2Maps[ iMap.name ] = iMap - return dName2Maps - - ## Return a list of Map instances with all the data contained in the table - # - # @return lMaps list of Map instances - # - def getListOfAllMaps( self ): - sqlCmd = "SELECT * FROM %s" % ( self._table ) - lMaps = self._iDb.getObjectListWithSQLCmd( sqlCmd, self._getInstanceToAdapt ) - return lMaps - - ## Give the end of map as integer - # - # @return end integer the end of map - # - def getEndFromSeqName(self, seqName): - sqlCmd = "SELECT end FROM %s WHERE chr = '%s'" % (self._table, seqName) - end = self._iDb.getIntegerWithSQLCmd(sqlCmd) - return end - - def _getInstanceToAdapt(self): - iMap = Map() - return iMap - - def _getTypeColumAttr2Get(self, name): - colum2Get = 'name' - type2Get = '%s' - attr2Get = name - return colum2Get, type2Get, attr2Get - - def _getTypeAndAttr2Insert(self, map): - type2Insert = ("'%s'","'%s'","'%d'","'%d'") - attr2Insert = (map.name, map.seqname, map.start, map.end) - return type2Insert, attr2Insert - - def _escapeAntislash(self, obj): - obj.name = obj.name.replace("\\", "\\\\") - obj.seqname = obj.seqname.replace("\\", "\\\\") |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/TableMatchAdaptator.py --- a/commons/core/sql/TableMatchAdaptator.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| @@ -1,100 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - - -from commons.core.sql.TableAdaptator import TableAdaptator -from commons.core.sql.ITableMatchAdaptator import ITableMatchAdaptator -from commons.core.coord.Match import Match - -## Adaptator for Match table -# -class TableMatchAdaptator( TableAdaptator, ITableMatchAdaptator ): - - ## Give a list of Match instances given a query name - # - # @param query string sequence name - # @return lMatches list of Match instances - # - def getMatchListFromQuery( self, query ): - sqlCmd = "SELECT * FROM %s WHERE query_name='%s';" % ( self._table, query ) - return self._iDb.getObjectListWithSQLCmd( sqlCmd, self._getInstanceToAdapt ) - - ## Give a list of Match instances having the same identifier - # - # @param id integer identifier number - # @return lMatch a list of Match instances - # - def getMatchListFromId( self, id ): - sqlCmd = "SELECT * FROM %s WHERE path='%d';" % ( self._table, id ) - lMatch = self._iDb.getObjectListWithSQLCmd( sqlCmd, self._getInstanceToAdapt ) - return lMatch - - ## Give a list of Match instances according to the given list of identifier numbers - # - # @param lId integer list - # @return lMatch a list of Match instances - # - def getMatchListFromIdList( self, lId ): - lMatch=[] - if lId == []: - return lMatch - sqlCmd = "select * from %s where path=%d" % (self._table, lId[0]) - for i in lId[1:]: - sqlCmd += " or path=%d" % (i) - sqlCmd += ";" - lMatch = self._iDb.getObjectListWithSQLCmd( sqlCmd, self._getInstanceToAdapt ) - return lMatch - - ## Give the data contained in the table as a list of Match instances - # - # @return lMatchs list of match instances - # - def getListOfAllMatches( self ): - sqlCmd = "SELECT * FROM %s" % ( self._table ) - lMatches = self._iDb.getObjectListWithSQLCmd( sqlCmd, self._getInstanceToAdapt ) - return lMatches - - def _getInstanceToAdapt(self): - iMatch = Match() - return iMatch - - def _getTypeAndAttr2Insert(self, match): - type2Insert = ("'%s'","'%d'","'%d'","'%d'","'%f'","'%f'","'%s'","'%d'","'%d'","'%d'","'%f'","'%g'","'%d'","'%f'","'%d'") - attr2Insert = ( match.range_query.seqname, match.range_query.start, \ - match.range_query.end, match.query_length, match.query_length_perc, \ - match.match_length_perc, match.range_subject.seqname, match.range_subject.start,\ - match.range_subject.end, match.subject_length, match.subject_length_perc, \ - match.e_value, match.score, match.identity, \ - match.id) - return type2Insert, attr2Insert - - def _escapeAntislash(self, obj): - obj.range_query.seqname = obj.range_query.seqname.replace("\\", "\\\\") - obj.range_subject.seqname = obj.range_subject.seqname.replace("\\", "\\\\") |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/TablePathAdaptator.py --- a/commons/core/sql/TablePathAdaptator.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| b'@@ -1,673 +0,0 @@\n-# Copyright INRA (Institut National de la Recherche Agronomique)\n-# http://www.inra.fr\n-# http://urgi.versailles.inra.fr\n-#\n-# This software is governed by the CeCILL license under French law and\n-# abiding by the rules of distribution of free software. You can use, \n-# modify and/ or redistribute the software under the terms of the CeCILL\n-# license as circulated by CEA, CNRS and INRIA at the following URL\n-# "http://www.cecill.info". \n-#\n-# As a counterpart to the access to the source code and rights to copy,\n-# modify and redistribute granted by the license, users are provided only\n-# with a limited warranty and the software\'s author, the holder of the\n-# economic rights, and the successive licensors have only limited\n-# liability. \n-#\n-# In this respect, the user\'s attention is drawn to the risks associated\n-# with loading, using, modifying and/or developing or reproducing the\n-# software by the user in light of its specific status of free software,\n-# that may mean that it is complicated to manipulate, and that also\n-# therefore means that it is reserved for developers and experienced\n-# professionals having in-depth computer knowledge. Users are therefore\n-# encouraged to load and test the software\'s suitability as regards their\n-# requirements in conditions enabling the security of their systems and/or \n-# data to be ensured and, more generally, to use and operate it in the \n-# same conditions as regards security. \n-#\n-# The fact that you are presently reading this means that you have had\n-# knowledge of the CeCILL license and that you accept its terms.\n-\n-\n-from commons.core.coord.Path import Path\n-from commons.core.coord.PathUtils import PathUtils\n-from commons.core.sql.TableAdaptator import TableAdaptator\n-from commons.core.sql.ITablePathAdaptator import ITablePathAdaptator\n-\n-\n-## Adaptator for a Path table\n-#\n-class TablePathAdaptator( TableAdaptator, ITablePathAdaptator ):\n-\n- ## Give a list of Path instances having the same identifier\n- #\n- # @param id integer identifier number\n- # @return lPath a list of Path instances\n- #\n- def getPathListFromId( self, id ):\n- sqlCmd = "SELECT * FROM %s WHERE path=\'%d\';" % ( self._table, id )\n- lPath = self._iDb.getObjectListWithSQLCmd( sqlCmd, self._getInstanceToAdapt )\n- return lPath\n- \n- ## Give a list of Path instances according to the given list of identifier numbers\n- #\n- # @param lId integer list \n- # @return lPath a list of Path instances\n- #\n- def getPathListFromIdList( self, lId ):\n- lPath=[]\n- if lId == []:\n- return lPath\n- sqlCmd = "select * from %s where path=%d" % (self._table, lId[0])\n- for i in lId[1:]:\n- sqlCmd += " or path=%d" % (i)\n- sqlCmd += ";"\n- lPath = self._iDb.getObjectListWithSQLCmd( sqlCmd, self._getInstanceToAdapt )\n- return lPath\n- \n- ## Give a list of Path instances having the same given query name\n- #\n- # @param query string name of the query \n- # @return lPath a list of Path instances\n- #\n- def getPathListFromQuery( self, query ):\n- lPath = self._getPathListFromTypeName("query", query)\n- return lPath\n- \n- ## Give a list of Path instances having the same given subject name\n- #\n- # @param subject string name of the subject \n- # @return lPath a list of Path instances\n- #\n- def getPathListFromSubject( self, subject ):\n- lPath = self._getPathListFromTypeName("subject", subject)\n- return lPath\n- \n- ## Give a list of the distinct subject names present in the table\n- #\n- # @return lDistinctSubjectNames string list\n- #\n- def getSubjectList(self):\n- lDistinctSubjectNames = self._getDistinctTypeNamesList("subject")\n- return lDistinctSubjectNames\n- \n- ## Give a list of the distinct query names present in the table\n- #\n- # @return lDistinctQueryNames string list\n- #\n- def ge'..b'TypeNamesList( self, type ):\n- sqlCmd = "SELECT DISTINCT %s_name FROM %s" % ( type, self._table )\n- lDistinctTypeNames = self._iDb.getStringListWithSQLCmd(sqlCmd)\n- return lDistinctTypeNames\n- \n- def _getPathsNbFromTypeName( self, type, typeName ):\n- sqlCmd = "SELECT COUNT(*) FROM %s WHERE %s_name=\'%s\'" % ( self._table, type, typeName )\n- pathNb = self._iDb.getIntegerWithSQLCmd( sqlCmd )\n- return pathNb\n- \n- def _getIdListFromTypeName( self, type, typeName ):\n- sqlCmd = "SELECT DISTINCT path FROM %s WHERE %s_name=\'%s\'" % ( self._table, type, typeName )\n- lId = self._iDb.getIntegerListWithSQLCmd( sqlCmd )\n- return lId\n- \n- def _getIdNbFromTypeName( self, type, typeName ):\n- sqlCmd = "SELECT COUNT( DISTINCT path ) FROM %s WHERE %s_name=\'%s\'" % ( self._table, type, typeName )\n- idNb = self._iDb.getIntegerWithSQLCmd( sqlCmd )\n- return idNb\n- \n- def _getTypeAndAttr2Insert(self, path):\n- type2Insert = ("\'%d\'", "\'%s\'", "\'%d\'", "\'%d\'", "\'%s\'", "\'%d\'", "\'%d\'", "\'%g\'", "\'%d\'", "\'%f\'")\n- if path.range_query.isOnDirectStrand():\n- queryStart = path.range_query.start\n- queryEnd = path.range_query.end\n- subjectStart = path.range_subject.start\n- subjectEnd = path.range_subject.end\n- else:\n- queryStart = path.range_query.end\n- queryEnd = path.range_query.start\n- subjectStart = path.range_subject.end\n- subjectEnd = path.range_subject.start\n- attr2Insert = ( path.id,\\\n- path.range_query.seqname,\\\n- queryStart,\\\n- queryEnd,\\\n- path.range_subject.seqname,\\\n- subjectStart,\\\n- subjectEnd,\\\n- path.e_value,\\\n- path.score,\\\n- path.identity\\\n- )\n- return type2Insert, attr2Insert\n- \n- def _getInstanceToAdapt(self):\n- iPath = Path()\n- return iPath\n- \n- def _escapeAntislash(self, obj):\n- obj.range_query.seqname = obj.range_query.seqname.replace("\\\\", "\\\\\\\\")\n- obj.range_subject.seqname = obj.range_subject.seqname.replace("\\\\", "\\\\\\\\")\n- \n- def _genSqlCmdForTmpTableAccordingToQueryName(self, queryName, tmpTable):\n- sqlCmd = ""\n- if queryName == "":\n- sqlCmd = "CREATE TABLE %s SELECT path, query_name, query_start, query_end, subject_name, subject_start, subject_end, e_value, score, (ABS(query_end-query_start)+1)*identity AS identity FROM %s" % (tmpTable, self._table)\n- else:\n- sqlCmd = "CREATE TABLE %s SELECT path, query_name, query_start, query_end, subject_name, subject_start, subject_end, e_value, score, (ABS(query_end-query_start)+1)*identity AS identity FROM %s WHERE query_name=\'%s\'" % (tmpTable, self._table, queryName)\n- return sqlCmd\n- \n- ## return a filtered list with only one unique occurrence of path of a given list\n- #\n- # @param lPath a list of Path instances\n- # @return lUniquePath a list of Path instances\n- #\n- def getListOfUniqueOccPath(self, lPath):\n- if len(lPath) < 2 :\n- return lPath\n- \n- sortedListPath = sorted(lPath, key=lambda iPath: ( iPath.range_query.getSeqname(), iPath.range_query.getStart(), iPath.range_query.getEnd(), iPath.range_subject.getSeqname(), iPath.range_subject.getStart(), iPath.range_subject.getEnd()))\n- lUniquePath = [] \n- for i in xrange(1, len(sortedListPath)):\n- previousPath = sortedListPath [i-1]\n- currentPath = sortedListPath [i]\n- if previousPath != currentPath:\n- lUniquePath.append(previousPath)\n- \n- if previousPath != currentPath:\n- lUniquePath.append(currentPath) \n- \n- return lUniquePath \n\\ No newline at end of file\n' |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/TableSeqAdaptator.py --- a/commons/core/sql/TableSeqAdaptator.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| @@ -1,185 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - - -import sys -from commons.core.sql.TableAdaptator import TableAdaptator -from commons.core.sql.ITableSeqAdaptator import ITableSeqAdaptator -from commons.core.coord.SetUtils import SetUtils -from commons.core.seq.Bioseq import Bioseq - - -## Adaptator for a Seq table -# -class TableSeqAdaptator( TableAdaptator, ITableSeqAdaptator ): - - ## Retrieve all the distinct accession names in a list. - # - # @return lAccessions list of accessions - # - def getAccessionsList( self ): - sqlCmd = "SELECT DISTINCT accession FROM %s;" % ( self._table ) - lAccessions = self._getStringListWithSQLCmd(sqlCmd) - return lAccessions - - ## Save sequences in a fasta file from a list of accession names. - # - # @param lAccessions list of accessions - # @param outFileName string Fasta file - # - def saveAccessionsListInFastaFile( self, lAccessions, outFileName ): - outFile = open( outFileName, "w" ) - for ac in lAccessions: - bs = self.getBioseqFromHeader( ac ) - bs.write(outFile) - outFile.close() - - ## Get a bioseq instance given its header - # - # @param header string name of the sequence ('accession' field in the 'seq' table) - # @return bioseq instance - # - def getBioseqFromHeader( self, header ): - sqlCmd = "SELECT * FROM %s WHERE accession='%s';" % ( self._table, header ) - self._iDb.execute( sqlCmd ) - res = self._iDb.fetchall() - return Bioseq( res[0][0], res[0][1] ) - - ## Retrieve the length of a sequence given its name. - # - # @param accession name of the sequence - # @return seqLength integer length of the sequence - # - def getSeqLengthFromAccession( self, accession ): - sqlCmd = 'SELECT length FROM %s WHERE accession="%s"' % ( self._table, accession ) - seqLength = self._iDb.getIntegerWithSQLCmd(sqlCmd) - return seqLength - - ## Retrieve the length of a sequence given its description. - # - # @param description of the sequence - # @return seqLength integer length of the sequence - # - def getSeqLengthFromDescription( self, description ): - sqlCmd = 'SELECT length FROM %s WHERE description="%s"' % ( self._table, description ) - seqLength = self._iDb.getIntegerWithSQLCmd(sqlCmd) - return seqLength - - ## Retrieve all the accessions with length in a list of tuples - # - # @return lAccessionLengthTuples list of tuples - # - def getAccessionAndLengthList(self): - sqlCmd = 'SELECT accession, length FROM %s' % self._table - self._iDb.execute(sqlCmd) - res = self._iDb.fetchall() - lAccessionLengthTuples = [] - for i in res: - lAccessionLengthTuples.append(i) - return lAccessionLengthTuples - - ## get subsequence according to given parameters - # - # @param accession - # @param start integer - # @param end integer - # @return bioseq.sequence string - # - def getSubSequence( self, accession, start, end ): - bs = Bioseq() - if start <= 0 or end <= 0: - print "ERROR with coordinates start=%i or end=%i" % ( start, end ) - sys.exit(1) - - if accession not in self.getAccessionsList(): - print "ERROR: accession '%s' absent from table '%s'" % ( accession, self._table ) - sys.exit(1) - - lengthAccession = self.getSeqLengthFromAccession( accession ) - if start > lengthAccession or end > lengthAccession: - print "ERROR: coordinates start=%i end=%i out of sequence '%s' range (%i bp)" % ( start, end, accession, lengthAccession ) - sys.exit(1) - - sqlCmd = "SELECT SUBSTRING(sequence,%i,%i) FROM %s WHERE accession='%s'" % ( min(start,end), abs(end-start)+ 1, self._table, accession ) - self._iDb.execute( sqlCmd ) - res = self._iDb.fetchall() - bs.setSequence( res[0][0] ) - if start > end: - bs.reverseComplement() - return bs.sequence - - ## get bioseq from given set list - # - # @param lSets set list of sets - # @return bioseq instance - # - def getBioseqFromSetList( self, lSets ): - header = "%s::%i %s " % ( lSets[0].name, lSets[0].id, lSets[0].seqname ) - sequence = "" - lSortedSets = SetUtils.getSetListSortedByIncreasingMinThenMax( lSets ) - if not lSets[0].isOnDirectStrand(): - lSortedSets.reverse() - for iSet in lSortedSets: - header += "%i..%i," % ( iSet.getStart(), iSet.getEnd() ) - sequence += self.getSubSequence( iSet.seqname, iSet.getStart(), iSet.getEnd() ) - return Bioseq( header[:-1], sequence ) - - ## Return True if the given accession is present in the table - # - def isAccessionInTable( self, name ): - sqlCmd = "SELECT accession FROM %s WHERE accession='%s'" % ( self._table, name ) - self._iDb.execute( sqlCmd ) - res = self._iDb.fetchall() - return bool(res) - - ## Retrieve all the distinct accession names in a fasta file. - # - # @param outFileName string Fasta file - # - def exportInFastaFile(self, outFileName ): - lAccessions = self.getAccessionsList() - self.saveAccessionsListInFastaFile( lAccessions, outFileName ) - - def _getStringListWithSQLCmd( self, sqlCmd ): - self._iDb.execute(sqlCmd) - res = self._iDb.fetchall() - lString = [] - for i in res: - lString.append(i[0]) - return lString - - def _getTypeAndAttr2Insert(self, bs): - type2Insert = ( "'%s'", "'%s'", "'%s'", "'%i'" ) - attr2Insert = (bs.header.split()[0], bs.sequence, bs.header, bs.getLength()) - return type2Insert, attr2Insert - - def _escapeAntislash(self, obj): - pass - |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/TableSetAdaptator.py --- a/commons/core/sql/TableSetAdaptator.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| b'@@ -1,215 +0,0 @@\n-# Copyright INRA (Institut National de la Recherche Agronomique)\n-# http://www.inra.fr\n-# http://urgi.versailles.inra.fr\n-#\n-# This software is governed by the CeCILL license under French law and\n-# abiding by the rules of distribution of free software. You can use, \n-# modify and/ or redistribute the software under the terms of the CeCILL\n-# license as circulated by CEA, CNRS and INRIA at the following URL\n-# "http://www.cecill.info". \n-#\n-# As a counterpart to the access to the source code and rights to copy,\n-# modify and redistribute granted by the license, users are provided only\n-# with a limited warranty and the software\'s author, the holder of the\n-# economic rights, and the successive licensors have only limited\n-# liability. \n-#\n-# In this respect, the user\'s attention is drawn to the risks associated\n-# with loading, using, modifying and/or developing or reproducing the\n-# software by the user in light of its specific status of free software,\n-# that may mean that it is complicated to manipulate, and that also\n-# therefore means that it is reserved for developers and experienced\n-# professionals having in-depth computer knowledge. Users are therefore\n-# encouraged to load and test the software\'s suitability as regards their\n-# requirements in conditions enabling the security of their systems and/or \n-# data to be ensured and, more generally, to use and operate it in the \n-# same conditions as regards security. \n-#\n-# The fact that you are presently reading this means that you have had\n-# knowledge of the CeCILL license and that you accept its terms.\n-\n-\n-from commons.core.sql.ITableSetAdaptator import ITableSetAdaptator\n-from commons.core.sql.TableAdaptator import TableAdaptator\n-from commons.core.coord.Set import Set\n-\n-\n-## Adaptator for a Set table\n-#\n-class TableSetAdaptator( TableAdaptator, ITableSetAdaptator ):\n- \n- ## Give a list of Set instances having a given seq name\n- #\n- # @param seqName string seq name\n- # @return lSet list of instances\n- #\n- def getListFromSeqName( self, seqName ):\n- sqlCmd = "SELECT * FROM %s" % (self._table)\n- colum2Get, type2Get, attr2Get = self._getTypeColumAttr2Get(seqName)\n- sqlCmd += " WHERE " + colum2Get\n- sqlCmd += " = "\n- sqlCmd = sqlCmd + type2Get\n- sqlCmd = sqlCmd % "\'" + attr2Get + "\'"\n- lSet = self._iDb.getObjectListWithSQLCmd( sqlCmd, self._getInstanceToAdapt )\n- return lSet\n- \n- ## Give a list of set instances overlapping a given region\n- #\n- # @param query string query name\n- # @param start integer start coordinate\n- # @param end integer end coordinate\n- # @return lSet list of set instances\n- #\n- def getListOverlappingCoord(self, query, start, end):\n- sqlCmd = \'select * from %s where chr="%s" and ((start between least(%d,%d) and greatest(%d,%d) or end between least(%d,%d) and greatest(%d,%d)) or (least(start,end)<=least(%d,%d) and greatest(start,end)>=greatest(%d,%d))) ;\' % (self._table, query, start, end, start, end, start, end, start, end, start, end, start, end)\n- lSet = self._iDb.getObjectListWithSQLCmd( sqlCmd, self._getInstanceToAdapt )\n- return lSet\n-\n- #TODO: to test !!!\n- ## Give a list of Set instances overlapping a given region\n- #\n- # @note whole chains are returned, even if only a fragment overlap with the given region\n- # @param query string query name\n- # @param start integer start coordinate\n- # @param end integer end coordinate\n- # @return lSets list of Path instances\n- #\n- def getChainListOverlappingCoord(self, query, start, end):\n- sqlCmd = "select distinct path from %s where chr=\'%s\' and ((start between least(%d,%d) and greatest(%d,%d) or end between least(%d,%d) and greatest(%d,%d)) or (least(start,end)<=least(%d,%d) and greatest(start,end)>=greatest(%d,%d)));" % (self._table, query,start,end,start,end,start,end,start,end,start,end,start,e'..b'lCmd)\n- return lDistinctContigNames\n- \n- ## Give a list of Set instances having a given seq name\n- #\n- # @param seqName string seq name\n- # @return lSet list of instances\n- #\n- def getSetListFromSeqName( self, seqName):\n- lSets = self.getListFromSeqName(seqName)\n- return lSets\n- \n- ## Give a set instances list with a given identifier number\n- #\n- # @param id integer identifier number\n- # @return lSet list of set instances\n- #\n- def getSetListFromId(self, id):\n- SQLCmd = "select * from %s where path=%d;" % (self._table, id)\n- return self._iDb.getObjectListWithSQLCmd( SQLCmd, self._getInstanceToAdapt )\n- \n- ## Give a set instances list with a list of identifier numbers\n- #\n- # @param lId integers list identifiers list numbers\n- # @return lSet list of set instances\n- # \n- def getSetListFromIdList(self,lId):\n- lSet = []\n- if lId == []:\n- return lSet\n- SQLCmd = "select * from %s where path=%d" % (self._table, lId[0])\n- for i in lId[1:]:\n- SQLCmd += " or path=%d" % (i)\n- SQLCmd += ";"\n- return self._iDb.getObjectListWithSQLCmd( SQLCmd, self._getInstanceToAdapt )\n- \n- ## Return a list of Set instances overlapping a given sequence\n- # \n- # @param seqName string sequence name\n- # @param start integer start coordinate\n- # @param end integer end coordinate\n- # @return lSet list of Set instances\n- #\n- def getSetListOverlappingCoord( self, seqName, start, end ):\n- lSet = self.getListOverlappingCoord( seqName, start, end )\n- return lSet\n- \n- ## Delete set corresponding to a given identifier number\n- #\n- # @param id integer identifier number\n- # \n- def deleteFromId(self, id):\n- sqlCmd = "delete from %s where path=%d;" % (self._table, id)\n- self._iDb.execute(sqlCmd)\n- \n- ## Delete set corresponding to a given list of identifier number\n- #\n- # @param lId integers list list of identifier number\n- # \n- def deleteFromIdList(self, lId):\n- if lId == []:\n- return\n- sqlCmd = "delete from %s where path=%d" % ( self._table, lId[0] )\n- for i in lId[1:]:\n- sqlCmd += " or path=%d"%(i)\n- sqlCmd += ";"\n- self._iDb.execute(sqlCmd)\n- \n- ## Join two set by changing id number of id1 and id2 set to the least of id1 and id2\n- #\n- # @param id1 integer id path number\n- # @param id2 integer id path number\n- # \n- def joinTwoSets(self, id1, id2):\n- if id1 < id2:\n- newId = id1\n- oldId = id2\n- else:\n- newId = id2\n- oldId = id1\n- sqlCmd = "UPDATE %s SET path=%d WHERE path=%d" % (self._table, newId, oldId)\n- self._iDb.execute(sqlCmd)\n- \n- ## Get a new id number\n- #\n- # @return new_id integer max_id + 1 \n- #\n- def getNewId(self):\n- sqlCmd = "select max(path) from %s;" % (self._table)\n- maxId = self._iDb.getIntegerWithSQLCmd(sqlCmd)\n- newId = int(maxId) + 1\n- return newId\n- \n- ## Give the data contained in the table as a list of Sets instances\n- #\n- # @return lSets list of set instances\n- #\n- def getListOfAllSets( self ):\n- return self.getListOfAllCoordObject()\n- \n- def _getInstanceToAdapt(self):\n- iSet = Set()\n- return iSet\n- \n- def _getTypeColumAttr2Get(self, contig):\n- colum2Get = \'chr\'\n- type2Get = \'%s\'\n- attr2Get = contig\n- return colum2Get, type2Get, attr2Get\n- \n- def _getTypeAndAttr2Insert(self, set):\n- type2Insert = ("\'%d\'","\'%s\'","\'%s\'","\'%d\'","\'%d\'")\n- attr2Insert = (set.id, set.name, set.seqname, set.start, set.end)\n- return type2Insert, attr2Insert\n-\n- def _escapeAntislash(self, obj):\n- obj.name = obj.name.replace("\\\\", "\\\\\\\\")\n- obj.seqname = obj.seqname.replace("\\\\", "\\\\\\\\")\n' |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/test/TestSuite_sql.py --- a/commons/core/sql/test/TestSuite_sql.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| b |
| @@ -1,68 +0,0 @@ -#!/usr/bin/env python - -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - - -import unittest -import sys -import Test_DbMySql -import Test_TableBinPathAdaptator -import Test_TableMapAdaptator -import Test_TableMatchAdaptator -import Test_TablePathAdaptator -import Test_TableSeqAdaptator -import Test_TableSetAdaptator -import Test_F_RepetJob -import Test_RepetJob -import Test_TableBinSetAdaptator - -def main(): - - TestSuite_sql = unittest.TestSuite() - - TestSuite_sql.addTest( unittest.makeSuite( Test_DbMySql.Test_DbMySql, "test" ) ) - TestSuite_sql.addTest( unittest.makeSuite( Test_TableBinPathAdaptator.Test_TableBinPathAdaptator, "test" ) ) - TestSuite_sql.addTest( unittest.makeSuite( Test_TableMapAdaptator.Test_TableMapAdaptator, "test" ) ) - TestSuite_sql.addTest( unittest.makeSuite( Test_TableMatchAdaptator.Test_TableMatchAdaptator, "test" ) ) - TestSuite_sql.addTest( unittest.makeSuite( Test_TableSetAdaptator.Test_TableSetAdaptator, "test" ) ) - TestSuite_sql.addTest( unittest.makeSuite( Test_TableSeqAdaptator.Test_TableSeqAdaptator, "test" ) ) - TestSuite_sql.addTest( unittest.makeSuite( Test_TableMatchAdaptator.Test_TableMatchAdaptator, "test" ) ) - TestSuite_sql.addTest( unittest.makeSuite( Test_TablePathAdaptator.Test_TablePathAdaptator, "test" ) ) - TestSuite_sql.addTest( unittest.makeSuite( Test_F_RepetJob.Test_F_RepetJob, "test" ) ) - TestSuite_sql.addTest( unittest.makeSuite( Test_RepetJob.Test_RepetJob, "test" ) ) - TestSuite_sql.addTest( unittest.makeSuite( Test_TableBinSetAdaptator.Test_TableBinSetAdaptator, "test" ) ) - - runner = unittest.TextTestRunner( sys.stderr, 2, 2 ) - runner.run( TestSuite_sql ) - - -if __name__ == "__main__": - main() |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/test/Test_DbFactory.py --- a/commons/core/sql/test/Test_DbFactory.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| @@ -1,63 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - -import os -import unittest -from commons.core.sql.DbFactory import DbFactory - -class Test_DbFactory( unittest.TestCase ): - - def test_createInstance (self): - dbInstance = DbFactory.createInstance() - expValue = None - obsValue = dbInstance - self.assertNotEquals(expValue, obsValue) - - def test_createInstance_with_config (self): - configFileName = "dummyConfigFileName.cfg" - configF = open(configFileName,"w") - configF.write("[repet_env]\n") - configF.write( "repet_host: %s\n" % ( os.environ["REPET_HOST"] ) ) - configF.write( "repet_user: %s\n" % ( os.environ["REPET_USER"] ) ) - configF.write( "repet_pw: %s\n" % ( os.environ["REPET_PW"] ) ) - configF.write( "repet_db: %s\n" % ( os.environ["REPET_DB"] ) ) - configF.write( "repet_port: %s\n" % ( os.environ["REPET_PORT"] ) ) - configF.close() - - dbInstance = DbFactory.createInstance(configFileName) - expValue = None - obsValue = dbInstance - self.assertNotEquals(expValue, obsValue) - os.remove(configFileName) - -test_suite = unittest.TestSuite() -test_suite.addTest( unittest.makeSuite( Test_DbFactory ) ) -if __name__ == "__main__": - unittest.TextTestRunner(verbosity=2).run( test_suite ) |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/test/Test_DbMySql.py --- a/commons/core/sql/test/Test_DbMySql.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| b'@@ -1,1554 +0,0 @@\n-# Copyright INRA (Institut National de la Recherche Agronomique)\n-# http://www.inra.fr\n-# http://urgi.versailles.inra.fr\n-#\n-# This software is governed by the CeCILL license under French law and\n-# abiding by the rules of distribution of free software. You can use, \n-# modify and/ or redistribute the software under the terms of the CeCILL\n-# license as circulated by CEA, CNRS and INRIA at the following URL\n-# "http://www.cecill.info". \n-#\n-# As a counterpart to the access to the source code and rights to copy,\n-# modify and redistribute granted by the license, users are provided only\n-# with a limited warranty and the software\'s author, the holder of the\n-# economic rights, and the successive licensors have only limited\n-# liability. \n-#\n-# In this respect, the user\'s attention is drawn to the risks associated\n-# with loading, using, modifying and/or developing or reproducing the\n-# software by the user in light of its specific status of free software,\n-# that may mean that it is complicated to manipulate, and that also\n-# therefore means that it is reserved for developers and experienced\n-# professionals having in-depth computer knowledge. Users are therefore\n-# encouraged to load and test the software\'s suitability as regards their\n-# requirements in conditions enabling the security of their systems and/or \n-# data to be ensured and, more generally, to use and operate it in the \n-# same conditions as regards security. \n-#\n-# The fact that you are presently reading this means that you have had\n-# knowledge of the CeCILL license and that you accept its terms.\n-\n-import unittest\n-import time\n-import os\n-from MySQLdb import ProgrammingError\n-from commons.core.sql.DbMySql import DbMySql\n-from commons.core.sql.DbMySql import TABLE_SCHEMA_DESCRIPTOR\n-from commons.core.sql.DbMySql import TABLE_TYPE_SYNONYMS\n-from commons.core.utils.FileUtils import FileUtils\n-from commons.core.coord.Path import Path\n-\n-class Test_DbMySql( unittest.TestCase ):\n- \n- def setUp( self ):\n- self._iDb = DbMySql( )\n- self._uniqId = "%s" % time.strftime("%Y%m%d%H%M%S")\n-\n- def tearDown( self ):\n- if self._iDb.db.open:\n- self._iDb.close()\n- self._iDb = None\n- \n- def test_execute_syntax_error(self):\n- expErrorMsg = "You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near \'CHAUD TABLES\' at line 1"\n- obsErrorMsg = ""\n- sqlCmd = "CHAUD TABLES"\n- try:\n- self._iDb.execute(sqlCmd)\n- except ProgrammingError as excep:\n- obsErrorMsg = excep.args[1]\n- \n- self.assertEquals(expErrorMsg, obsErrorMsg)\n-\n- def test_execute_with_1_retry(self):\n- tableName = "dummyTable%s" % self._uniqId\n- sqlCmd = "CREATE TABLE %s (dummyColumn varchar(255))" % tableName\n- self._iDb.close()\n- self._iDb.execute(sqlCmd)\n- self.assertTrue(self._iDb.doesTableExist(tableName))\n- self._iDb.dropTable(tableName)\n-\n- def test_setAttributesFromConfigFile(self):\n- expHost = "dummyHost"\n- expUser = "dummyUser"\n- expPw = "dummyPw"\n- expDb = "dummyDb"\n- expPort = 1000\n- \n- configFileName = "dummyConfigFileName.cfg"\n- f = open( configFileName, "w" )\n- f.write("[repet_env]\\n")\n- f.write("repet_host: " + expHost + "\\n")\n- f.write("repet_user: " + expUser + "\\n")\n- f.write("repet_pw: " + expPw + "\\n")\n- f.write("repet_db: " + expDb + "\\n")\n- f.write("repet_port: " + str(expPort) + "\\n")\n- f.close()\n- \n- self._iDb.setAttributesFromConfigFile(configFileName)\n- \n- obsHost = self._iDb.host\n- obsUser = self._iDb.user\n- obsPw = self._iDb.passwd\n- obsDb = self._iDb.dbname\n- obsPort = self._iDb.port\n- \n- os.remove(configFileName)\n- \n- self.asse'..b'l_r4.3: 3.73%; TermRepeats: non-termLTR: 1701; SSRCoverage=0.14<0.75)\\n")\n- \n- self._iDb.createTable(tableName, "classif", fileName)\n- self.assertTrue(self._iDb.doesTableExist(tableName))\n- \n- expColumnNb = 8\n- sqlCmd = "DESC %s;" % tableName\n- self._iDb.execute(sqlCmd)\n- res = self._iDb.fetchall()\n- obsColumnNb = len(res)\n- self.assertEquals(expColumnNb, obsColumnNb)\n- \n- expSize = 3\n- obsSize = self._iDb.getSize(tableName)\n- self.assertEquals(expSize, obsSize)\n- \n- expLIndex = ["iseq_name", "istatus", "iclass", "iorder", "icomp"]\n- sqlCmd = "SHOW INDEX FROM %s" % tableName\n- self._iDb.execute(sqlCmd)\n- res = self._iDb.cursor.fetchall()\n- obsLIndex = []\n- for tuple in res:\n- obsLIndex.append(tuple[2])\n- self.assertEquals(expLIndex, obsLIndex)\n- \n- self._iDb.dropTable(tableName)\n- os.remove(fileName)\n- \n- def test_createClassifIndex(self):\n- tableName = "dummyclassifTable%s" % self._uniqId\n- sqlCmd = "CREATE TABLE %s (seq_name varchar(255), length int unsigned, strand char, status varchar(255), class_classif varchar(255), order_classif varchar(255), completeness varchar(255), evidences text);" % tableName\n- self._iDb.execute(sqlCmd)\n- expLIndex = ["iseq_name", "istatus", "iclass", "iorder", "icomp"]\n- \n- self._iDb.createIndex(tableName, "classif")\n- \n- sqlCmd = "SHOW INDEX FROM %s" % tableName\n- self._iDb.execute(sqlCmd)\n- res = self._iDb.cursor.fetchall()\n- \n- obsLIndex = []\n- for tuple in res:\n- obsLIndex.append(tuple[2])\n- self.assertEquals(expLIndex, obsLIndex)\n- self._iDb.dropTable(tableName)\n-\n- def test_createBinPathTable(self):\n- pathFileName = "dummy.path"\n- with open(pathFileName, "w") as pathF:\n- pathF.write("1\\tqry\\t1\\t100\\tsbj\\t1\\t100\\t1e-123\\t136\\t98.4\\n")\n- pathF.write("2\\tqry\\t500\\t401\\tsbj\\t1\\t100\\t1e-152\\t161\\t98.7\\n")\n- \n- expPathTuple1 = (1, 1000000, "qry", 1, 100, 1)\n- expPathTuple2 = (2, 1000000, "qry", 401, 500, 1) # change coordinates\n- expTPathTuples = (expPathTuple1, expPathTuple2)\n- \n- pathTableName = "dummy_path"\n- idxTableName = "dummy_path_idx"\n- self._iDb.createTable(pathTableName, "path", pathFileName)\n- self._iDb.createBinPathTable(pathTableName, True)\n- \n- sqlCmd = "SELECT * FROM %s" % idxTableName\n- self._iDb.execute(sqlCmd)\n- obsTPathTuples = self._iDb.fetchall()\n- \n- self._iDb.dropTable(pathTableName)\n- self._iDb.dropTable(idxTableName)\n- os.remove(pathFileName)\n- \n- self.assertEquals(expTPathTuples, obsTPathTuples)\n-\n- def test_createBinSetTable(self):\n- setFileName = "dummy.set"\n- with open(setFileName, "w") as setF:\n- setF.write("1\\tseq1\\tchr1\\t1900\\t3900\\n")\n- setF.write("2\\tseq2\\tchr1\\t2\\t9\\n")\n- setF.write("3\\tseq3\\tchr1\\t8\\t13\\n")\n- \n- expTuple = ((1L, 10000.0, \'chr1\', 1900L, 3900L, 1L), (2L, 1000.0, \'chr1\', 2L, 9L, 1L), (3L, 1000.0, \'chr1\', 8L, 13L, 1L))\n- \n- setTableName = "dummy_set"\n- idxTableName = "dummy_set_idx"\n- self._iDb.createTable(setTableName, "set", setFileName)\n- self._iDb.createBinSetTable(setTableName, True)\n- \n- sqlCmd = "SELECT * FROM %s" % idxTableName\n- self._iDb.execute(sqlCmd)\n- obsTuple = self._iDb.fetchall()\n- \n- self._iDb.dropTable(setTableName)\n- self._iDb.dropTable(idxTableName)\n- os.remove(setFileName)\n- \n- self.assertEquals(expTuple, obsTuple)\n-\n- def _getInstanceToAdapt(self):\n- iPath = Path()\n- return iPath\n- \n-if __name__ == "__main__":\n- unittest.main()\n\\ No newline at end of file\n' |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/test/Test_DbSQLite.py --- a/commons/core/sql/test/Test_DbSQLite.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| @@ -1,162 +0,0 @@ -# Copyright INRA (Institut National de la Recherche Agronomique) -# http://www.inra.fr -# http://urgi.versailles.inra.fr -# -# This software is governed by the CeCILL license under French law and -# abiding by the rules of distribution of free software. You can use, -# modify and/ or redistribute the software under the terms of the CeCILL -# license as circulated by CEA, CNRS and INRIA at the following URL -# "http://www.cecill.info". -# -# As a counterpart to the access to the source code and rights to copy, -# modify and redistribute granted by the license, users are provided only -# with a limited warranty and the software's author, the holder of the -# economic rights, and the successive licensors have only limited -# liability. -# -# In this respect, the user's attention is drawn to the risks associated -# with loading, using, modifying and/or developing or reproducing the -# software by the user in light of its specific status of free software, -# that may mean that it is complicated to manipulate, and that also -# therefore means that it is reserved for developers and experienced -# professionals having in-depth computer knowledge. Users are therefore -# encouraged to load and test the software's suitability as regards their -# requirements in conditions enabling the security of their systems and/or -# data to be ensured and, more generally, to use and operate it in the -# same conditions as regards security. -# -# The fact that you are presently reading this means that you have had -# knowledge of the CeCILL license and that you accept its terms. - - -import unittest -import time -from commons.core.sql.DbSQLite import DbSQLite - -class Test_DbSQLite(unittest.TestCase): - - def setUp( self ): - self._iDb = DbSQLite("test.db") - self._uniqId = "%s" % time.strftime("%Y%m%d%H%M%S") - - def tearDown( self ): - if self._iDb.open(): - self._iDb.close() - self._iDb.delete() - self._iDb = None - - def test_open_True(self): - self._iDb.close() - self.assertTrue( self._iDb.open(1) ) - - def test_open_False(self): - self._iDb.close() - self._iDb.host = "/toto/toto.db" - self.assertFalse( self._iDb.open(1) ) - self._iDb.host = "test.db" - - def test_updateInfoTable(self): - tableName = "dummyTable" + self._uniqId - info = "Table_for_test" - - self._iDb.updateInfoTable(tableName, info) - - sqlCmd = 'SELECT file FROM info_tables WHERE name = "%s"' % ( tableName ) - self._iDb.execute( sqlCmd ) - results = self._iDb.fetchall() - obsResult = False - if (info,) in results: - obsResult = True - sqlCmd = 'DELETE FROM info_tables WHERE name = "%s"' % ( tableName ) - self._iDb.execute( sqlCmd ) - - self.assertTrue( obsResult ) - - def test_doesTableExist_True(self): - tableName = "dummyTable" + self._uniqId - sqlCmd = "CREATE TABLE %s ( dummyColumn varchar(255) )" % ( tableName ) - self._iDb.execute( sqlCmd ) - self.assertTrue( self._iDb.doesTableExist(tableName) ) - - def test_dropTable(self): - tableName = "dummyTable" + self._uniqId - sqlCmd = "CREATE TABLE %s ( dummyColumn varchar(255) )" % tableName - self._iDb.execute( sqlCmd ) - sqlCmd = "CREATE TABLE info_tables ( name varchar(255), file varchar(255) )" - self._iDb.execute( sqlCmd ) - sqlCmd = 'INSERT INTO info_tables VALUES ("%s","")' % tableName - self._iDb.execute( sqlCmd ) - - self._iDb.dropTable(tableName) - self.assertFalse( self._iDb.doesTableExist(tableName) ) - - def test_doesTableExist_False(self): - tableName = "dummyTable" + self._uniqId - self.assertFalse( self._iDb.doesTableExist(tableName) ) - - def test_createJobTable_is_table_created(self): - self._iDb.createTable("dummyJobTable", "jobs") - isTableCreated = self._iDb.doesTableExist("dummyJobTable") - self.assertTrue(isTableCreated) - - def test_createJobTable_field_list(self): - self._iDb.createTable("dummyJobTable", "jobs") - obsLFiled = self._iDb.getFieldList("dummyJobTable") - expLField = ["jobid", "jobname", "groupid", "command", "launcher", "queue", "status", "time", "node"] - self.assertEquals(expLField, obsLFiled) - - def test_createTable(self): - tableName = "dummyJobTable" + self._uniqId - self._iDb.createTable(tableName, "job") - obsLFiled = self._iDb.getFieldList(tableName) - expLField = ["jobid", "jobname", "groupid", "command", "launcher", "queue", "status", "time", "node"] - self.assertEquals(expLField, obsLFiled) - - def test_createTable_with_overwrite_Job(self): - tableName = "dummyJobTable" + self._uniqId - sqlCmd = "CREATE TABLE %s ( dummyColumn varchar(255) )" % tableName - self._iDb.execute( sqlCmd ) - sqlCmd = "CREATE TABLE info_tables ( name varchar(255), file varchar(255) )" - self._iDb.execute( sqlCmd ) - sqlCmd = 'INSERT INTO info_tables VALUES ("%s","")' % tableName - self._iDb.execute( sqlCmd ) - - self._iDb.createTable(tableName, "job", True) - obsLFiled = self._iDb.getFieldList(tableName) - expLField = ["jobid", "jobname", "groupid", "command", "launcher", "queue", "status", "time", "node"] - self.assertEquals(expLField, obsLFiled) - - def test_getSize_empty_table(self): - tableName = "dummyJobTable" + self._uniqId - sqlCmd = "CREATE TABLE %s ( dummyColumn varchar(255) )" % ( tableName ) - self._iDb.execute( sqlCmd ) - expSize = 0 - obsSize = self._iDb.getSize(tableName) - self.assertEquals( expSize, obsSize ) - - def test_getSize_one_rows(self): - tableName = "dummyJobTable" + self._uniqId - sqlCmd = "CREATE TABLE %s ( dummyColumn varchar(255) )" % ( tableName ) - self._iDb.execute( sqlCmd ) - sqlCmd = "INSERT INTO %s (dummyColumn) VALUES ('toto')" % tableName - self._iDb.execute( sqlCmd ) - expSize = 1 - obsSize = self._iDb.getSize(tableName) - self.assertEquals( expSize, obsSize ) - - def test_isEmpty_True(self): - tableName = "dummyTable" + self._uniqId - sqlCmd = "CREATE TABLE %s ( dummyColumn varchar(255) )" % ( tableName ) - self._iDb.execute( sqlCmd ) - self.assertTrue(self._iDb.isEmpty(tableName)) - - def test_isEmpty_False(self): - tableName = "dummyTable" + self._uniqId - sqlCmd = "CREATE TABLE %s ( dummyColumn varchar(255) )" % (tableName) - self._iDb.execute(sqlCmd) - sqlCmd = "INSERT INTO %s (dummyColumn) VALUES ('toto')" % tableName - self._iDb.execute(sqlCmd) - self.assertFalse(self._iDb.isEmpty(tableName)) - -if __name__ == "__main__": - unittest.main() \ No newline at end of file |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/test/Test_F_JobAdaptator.py --- a/commons/core/sql/test/Test_F_JobAdaptator.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| @@ -1,91 +0,0 @@ -from commons.core.launcher.WriteScript import WriteScript -from commons.core.sql.Job import Job -from commons.core.sql.DbFactory import DbFactory -from commons.core.sql.TableJobAdaptatorFactory import TableJobAdaptatorFactory -import sys -import stat -import os -import time -import unittest -import glob - -class Test_F_TableJobAdaptator(unittest.TestCase): - - def setUp(self): - self._jobTableName = "dummyJobTable" - self._iJA = TableJobAdaptatorFactory.createJobInstance() - - def tearDown(self): - pass - - def test_submitJob(self): - job1 = self._createJobInstance("job1") - self._createLauncherFile(job1, self._iJA) - job2 = self._createJobInstance("job2") - self._createLauncherFile(job2, self._iJA) - job3 = self._createJobInstance("job3") - self._createLauncherFile(job3, self._iJA) - - self._iJA.submitJob( job1, maxNbWaitingJobs=3, checkInterval=5, verbose=0 ) - self._iJA.submitJob( job2, maxNbWaitingJobs=3, checkInterval=5, verbose=0 ) - self._iJA.submitJob( job3, maxNbWaitingJobs=3, checkInterval=5, verbose=0 ) - - time.sleep(120) - - expErrorFilePrefix1 = job1.jobname + ".e" - expOutputFilePrefix1 = job1.jobname + ".o" - expErrorFilePrefix2 = job2.jobname + ".e" - expOutputFilePrefix2 = job2.jobname + ".o" - expErrorFilePrefix3 = job3.jobname + ".e" - expOutputFilePrefix3 = job3.jobname + ".o" - - lErrorFiles1 = glob.glob(expErrorFilePrefix1 + "*") - lOutputFiles1 = glob.glob(expOutputFilePrefix1 + "*") - lErrorFiles2 = glob.glob(expErrorFilePrefix2 + "*") - lOutputFiles2 = glob.glob(expOutputFilePrefix2 + "*") - lErrorFiles3 = glob.glob(expErrorFilePrefix3 + "*") - lOutputFiles3 = glob.glob(expOutputFilePrefix3 + "*") - - isLErrorFileNotEmpty1 = (len(lErrorFiles1) != 0) - isLOutputFileNotEmpty1 = (len(lOutputFiles1) != 0) - isLErrorFileNotEmpty2 = (len(lErrorFiles2) != 0) - isLOutputFileNotEmpty2 = (len(lOutputFiles2) != 0) - isLErrorFileNotEmpty3 = (len(lErrorFiles3) != 0) - isLOutputFileNotEmpty3 = (len(lOutputFiles3) != 0) - - os.system("rm launcherFileTest*.py *.e* *.o*") - self.assertTrue(isLErrorFileNotEmpty1 and isLOutputFileNotEmpty1) - self.assertTrue(isLErrorFileNotEmpty2 and isLOutputFileNotEmpty2) - self.assertTrue(isLErrorFileNotEmpty3 and isLOutputFileNotEmpty3) - - def test_submit_and_waitJobGroup(self): - iJob = self._createJobInstance("test") - self._createLauncherFile(iJob, self._iJA) - - self._iJA.submitJob( iJob, maxNbWaitingJobs=3, checkInterval=5, verbose=0 ) - self._iJA.waitJobGroup(iJob.groupid, 0, 2) - - expErrorFilePrefix1 = iJob.jobname + ".e" - expOutputFilePrefix1 = iJob.jobname + ".o" - - lErrorFiles1 = glob.glob(expErrorFilePrefix1 + "*") - lOutputFiles1 = glob.glob(expOutputFilePrefix1 + "*") - - isLErrorFileExist = (len(lErrorFiles1) != 0) - isLOutputFileExist = (len(lOutputFiles1) != 0) - os.system("rm launcherFileTest*.py *.e* *.o*") - self.assertTrue(isLErrorFileExist and isLOutputFileExist) - - def _createJobInstance(self, name): - lResources = [] - if os.environ.get("HOSTNAME") == "compute-2-46.local": - lResources.append("test=TRUE") - return Job(0, name, "test", "", "log = os.system(\"date;sleep 5;date\")", "%s/launcherFileTest_%s.py" % (os.getcwd(), name), lResources=lResources) - - def _createLauncherFile(self, iJob, iJA): - iWriteScript = WriteScript(iJob, iJA, os.getcwd(), os.getcwd(), False, True) - iWriteScript.run(iJob.command, "", iJob.launcher) - os.chmod(iJob.launcher, stat.S_IRWXU+stat.S_IRWXG+stat.S_IRWXO) - -if __name__ == "__main__": - unittest.main() |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/test/Test_F_TableJobAdaptator.py --- a/commons/core/sql/test/Test_F_TableJobAdaptator.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| @@ -1,185 +0,0 @@ -from commons.core.launcher.WriteScript import WriteScript -from commons.core.sql.Job import Job -from commons.core.sql.DbFactory import DbFactory -from commons.core.sql.TableJobAdaptatorFactory import TableJobAdaptatorFactory -import sys -import stat -import os -import time -import unittest -import glob - -class Test_F_TableJobAdaptator(unittest.TestCase): - - def setUp(self): - self._jobTableName = "dummyJobTable" - self._db = DbFactory.createInstance() - self._iTJA = TableJobAdaptatorFactory.createInstance(self._db, self._jobTableName) - - def tearDown(self): - self._db.dropTable(self._jobTableName) - self._db.close() - - def test_submitJob_with_multiple_jobs(self): - self._db.createTable(self._jobTableName, "jobs", overwrite = True) - job1 = _createJobInstance("job1") - _createLauncherFile(job1, self._iTJA) - job2 = _createJobInstance("job2") - _createLauncherFile(job2, self._iTJA) - job3 = _createJobInstance("job3") - _createLauncherFile(job3, self._iTJA) - - self._iTJA.submitJob( job1, maxNbWaitingJobs=3, checkInterval=5, verbose=0 ) - self._iTJA.submitJob( job2, maxNbWaitingJobs=3, checkInterval=5, verbose=0 ) - self._iTJA.submitJob( job3, maxNbWaitingJobs=3, checkInterval=5, verbose=0 ) - - time.sleep(120) - - expJobStatus = "finished" - obsJobStatus1 = self._iTJA.getJobStatus(job1) - obsJobStatus2 = self._iTJA.getJobStatus(job2) - obsJobStatus3 = self._iTJA.getJobStatus(job3) - - self.assertEquals(expJobStatus, obsJobStatus1) - self.assertEquals(expJobStatus, obsJobStatus2) - self.assertEquals(expJobStatus, obsJobStatus3) - - expErrorFilePrefix1 = job1.jobname + ".e" - expOutputFilePrefix1 = job1.jobname + ".o" - expErrorFilePrefix2 = job2.jobname + ".e" - expOutputFilePrefix2 = job2.jobname + ".o" - expErrorFilePrefix3 = job3.jobname + ".e" - expOutputFilePrefix3 = job3.jobname + ".o" - - lErrorFiles1 = glob.glob(expErrorFilePrefix1 + "*") - lOutputFiles1 = glob.glob(expOutputFilePrefix1 + "*") - lErrorFiles2 = glob.glob(expErrorFilePrefix2 + "*") - lOutputFiles2 = glob.glob(expOutputFilePrefix2 + "*") - lErrorFiles3 = glob.glob(expErrorFilePrefix3 + "*") - lOutputFiles3 = glob.glob(expOutputFilePrefix3 + "*") - - isLErrorFileNotEmpty1 = (len(lErrorFiles1) != 0) - isLOutputFileNotEmpty1 = (len(lOutputFiles1) != 0) - isLErrorFileNotEmpty2 = (len(lErrorFiles2) != 0) - isLOutputFileNotEmpty2 = (len(lOutputFiles2) != 0) - isLErrorFileNotEmpty3 = (len(lErrorFiles3) != 0) - isLOutputFileNotEmpty3 = (len(lOutputFiles3) != 0) - - os.system("rm launcherFileTest*.py *.e* *.o*") - self.assertTrue(isLErrorFileNotEmpty1 and isLOutputFileNotEmpty1) - self.assertTrue(isLErrorFileNotEmpty2 and isLOutputFileNotEmpty2) - self.assertTrue(isLErrorFileNotEmpty3 and isLOutputFileNotEmpty3) - - def test_submitJob_job_already_submitted(self): - self._db.createTable(self._jobTableName, "jobs", overwrite = True) - iJob = _createJobInstance("job") - self._iTJA.recordJob(iJob) - - isSysExitRaised = False - try: - self._iTJA.submitJob(iJob) - except SystemExit: - isSysExitRaised = True - self.assertTrue(isSysExitRaised) - - def test_waitJobGroup_with_error_job_maxRelaunch_two(self): - self._db.createTable(self._jobTableName, "jobs", overwrite = True) - iJob = _createJobInstance("job") - _createLauncherFile(iJob, self._iTJA) - - self._iTJA.recordJob(iJob) - self._iTJA.changeJobStatus(iJob, "error") - - self._iTJA.waitJobGroup(iJob.groupid, 0, 2) - - time.sleep(120) - - expJobStatus = "finished" - obsJobStatus1 = self._iTJA.getJobStatus(iJob) - - self.assertEquals(expJobStatus, obsJobStatus1) - - expErrorFilePrefix1 = iJob.jobname + ".e" - expOutputFilePrefix1 = iJob.jobname + ".o" - - lErrorFiles1 = glob.glob(expErrorFilePrefix1 + "*") - lOutputFiles1 = glob.glob(expOutputFilePrefix1 + "*") - - isLErrorFileNotEmpty1 = (len(lErrorFiles1) != 0) - isLOutputFileNotEmpty1 = (len(lOutputFiles1) != 0) - - self._iTJA.removeJob(iJob) - os.system("rm launcherFileTest*.py *.e* *.o*") - self.assertTrue(isLErrorFileNotEmpty1 and isLOutputFileNotEmpty1) - -class Test_F_TableJobAdaptator_SGE(unittest.TestCase): - - def setUp(self): - if os.environ["REPET_JOB_MANAGER"].lower() != "sge": - print "ERROR: jobs manager is not SGE: REPET_JOB_MANAGER = %s." % os.environ["REPET_JOB_MANAGER"] - sys.exit(0) - self._jobTableName = "dummyJobTable" - self._db = DbFactory.createInstance() - self._db.createTable(self._jobTableName, "jobs", overwrite = True) - self._iTJA = TableJobAdaptatorFactory.createInstance(self._db, self._jobTableName) - self._iJob = _createJobInstance("job") - _createLauncherFile(self._iJob, self._iTJA) - - def tearDown(self): - self._db.dropTable(self._jobTableName) - self._db.close() - - def test_waitJobGroup_with_several_nbTimeOut_waiting(self): - self._iTJA.recordJob(self._iJob) - self._iTJA.changeJobStatus(self._iJob, "running") - - expMsg = "ERROR: job '%s', supposedly still running, is not handled by SGE anymore\n" % self._iJob.jobid - - obsError = "obsError.txt" - obsErrorHandler = open(obsError, "w") - stderrRef = sys.stderr - sys.stderr = obsErrorHandler - - isSysExitRaised = False - try: - self._iTJA.waitJobGroup(self._iJob.groupid, timeOutPerJob = 3) - except SystemExit: - isSysExitRaised = True - - obsErrorHandler.close() - - obsErrorHandler = open(obsError, "r") - obsMsg = obsErrorHandler.readline() - obsErrorHandler.close() - - sys.stderr = stderrRef - os.remove(obsError) - os.system("rm launcherFileTest*.py") - self.assertTrue(isSysExitRaised) - self.assertEquals(expMsg, obsMsg) - - def test_isJobStillHandledBySge_True(self): - self._iTJA.submitJob(self._iJob) - isJobHandledBySge = self._iTJA.isJobStillHandledBySge(self._iJob.jobid, self._iJob.jobname) - os.system("rm launcherFileTest*.py") - self.assertTrue(isJobHandledBySge) - - def test_isJobStillHandledBySge_False(self): - self._iTJA.recordJob(self._iJob) - isJobHandledBySge = self._iTJA.isJobStillHandledBySge(self._iJob.jobid, self._iJob.jobname) - os.system("rm launcherFileTest*.py") - self.assertFalse(isJobHandledBySge) - -def _createJobInstance(name): - lResources = [] - if os.environ.get("HOSTNAME") == "compute-2-46.local": - lResources.append("test=TRUE") - return Job(0, name, "test", "", "log = os.system(\"date;sleep 5;date\")", "%s/launcherFileTest_%s.py" % (os.getcwd(), name), lResources=lResources) - -def _createLauncherFile(iJob, iTJA): - iWriteScript = WriteScript(iJob, iTJA, os.getcwd(), os.getcwd()) - iWriteScript.run(iJob.command, "", iJob.launcher) - os.chmod(iJob.launcher, stat.S_IRWXU+stat.S_IRWXG+stat.S_IRWXO) - -if __name__ == "__main__": - unittest.main() |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/test/Test_Job.py --- a/commons/core/sql/test/Test_Job.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| b |
| @@ -1,30 +0,0 @@ -import unittest -from commons.core.sql.Job import Job - -class Test_Job(unittest.TestCase): - - def test__eq__(self): - self._job = Job(jobid=0, jobname="test", groupid="test", queue="test",command="test", launcherFile="test", node="test", lResources="mem_free=1G" ) - o = Job(jobid=0, jobname="test", groupid="test", queue="test",command="test", launcherFile="test", node="test", lResources="mem_free=1G" ) - self.assertEqual( self._job, o ) # same data - o = Job(jobid=1, jobname="test", groupid="test", queue="test",command="test", launcherFile="test", node="test", lResources="mem_free=1G" ) - self.assertNotEqual( self._job, o ) # different jobid - o = Job(jobid=0, jobname="test1", groupid="test", queue="test",command="test", launcherFile="test", node="test", lResources="mem_free=1G" ) - self.assertNotEqual( self._job, o ) # different jobname - o = Job(jobid=0, jobname="test", groupid="test1", queue="test",command="test", launcherFile="test", node="test", lResources="mem_free=1G" ) - self.assertNotEqual( self._job, o ) # different groupid - o = Job(jobid=0, jobname="test", groupid="test", queue="test1",command="test", launcherFile="test", node="test", lResources="mem_free=1G" ) - self.assertNotEqual( self._job, o ) # different queue - o = Job(jobid=0, jobname="test", groupid="test", queue="test",command="test1", launcherFile="test", node="test", lResources="mem_free=1G" ) - self.assertNotEqual( self._job, o ) # different command - o = Job(jobid=0, jobname="test", groupid="test", queue="test",command="test", launcherFile="test1", node="test", lResources="mem_free=1G" ) - self.assertNotEqual( self._job, o ) # different launcherFile - o = Job(jobid=0, jobname="test", groupid="test", queue="test",command="test", launcherFile="test", node="test1", lResources="mem_free=1G" ) - self.assertNotEqual( self._job, o ) # different node - o = Job(jobid=0, jobname="test", groupid="test", queue="test",command="test", launcherFile="test", node="test", lResources="mem_free=2G" ) - self.assertNotEqual( self._job, o ) # different lResources - o = Job(jobid=0, jobname="test", groupid="test", queue="test",command="test", launcherFile="test", node="test", lResources="mem_free=1G", parallelEnvironment="multithread 6" ) - self.assertNotEqual( self._job, o ) # different parallelEnvironment - -if __name__ == "__main__": - unittest.main() \ No newline at end of file |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/test/Test_TableBinPathAdaptator.py --- a/commons/core/sql/test/Test_TableBinPathAdaptator.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| b'@@ -1,1244 +0,0 @@\n-# Copyright INRA (Institut National de la Recherche Agronomique)\n-# http://www.inra.fr\n-# http://urgi.versailles.inra.fr\n-#\n-# This software is governed by the CeCILL license under French law and\n-# abiding by the rules of distribution of free software. You can use, \n-# modify and/ or redistribute the software under the terms of the CeCILL\n-# license as circulated by CEA, CNRS and INRIA at the following URL\n-# "http://www.cecill.info". \n-#\n-# As a counterpart to the access to the source code and rights to copy,\n-# modify and redistribute granted by the license, users are provided only\n-# with a limited warranty and the software\'s author, the holder of the\n-# economic rights, and the successive licensors have only limited\n-# liability. \n-#\n-# In this respect, the user\'s attention is drawn to the risks associated\n-# with loading, using, modifying and/or developing or reproducing the\n-# software by the user in light of its specific status of free software,\n-# that may mean that it is complicated to manipulate, and that also\n-# therefore means that it is reserved for developers and experienced\n-# professionals having in-depth computer knowledge. Users are therefore\n-# encouraged to load and test the software\'s suitability as regards their\n-# requirements in conditions enabling the security of their systems and/or \n-# data to be ensured and, more generally, to use and operate it in the \n-# same conditions as regards security. \n-#\n-# The fact that you are presently reading this means that you have had\n-# knowledge of the CeCILL license and that you accept its terms.\n-\n-import unittest\n-import os\n-import time\n-from commons.core.sql.TableBinPathAdaptator import TableBinPathAdaptator\n-from commons.core.coord.Path import Path\n-from commons.core.coord.Set import Set\n-from commons.core.sql.DbFactory import DbFactory\n-\n-class Test_TableBinPathAdaptator( unittest.TestCase ):\n- \n- def setUp( self ):\n- self._uniqId = "%s_%s" % (time.strftime("%Y%m%d%H%M%S") , os.getpid())\n- self._db = DbFactory.createInstance()\n- self._table = "dummyPathTable_%s" % self._uniqId\n- self._table_idx = "dummyPathTable_%s_idx" % self._uniqId\n- \n- def tearDown( self ):\n- self._db.dropTable(self._table)\n- self._db.dropTable(self._table_idx)\n- self._db.close()\n- \n- #TODO: strand ?!? How does it work ?\n- def test_insert_QryRevSbjDir( self ):\n- tuple = ("1", "chr1", "10", "25", "TE1", "11", "17", "1e-18", "20", "87.4")\n- p1 = Path()\n- p1.setFromTuple(tuple)\n-\n- tuple = ("1", "chr1", "250", "100", "TE1", "11", "17", "1e-18", "20", "87.4")\n- p2 = Path()\n- p2.setFromTuple(tuple)\n- \n- tuple = ("2", "chr1", "15", "30", "TE2", "10", "13", "5e-24", "34", "93.1")\n- p3 = Path()\n- p3.setFromTuple(tuple)\n- \n- tuple = ("4", "chr5", "140", "251", "TE5", "140", "251", "2e-14", "14", "73.1")\n- p4 = Path()\n- p4.setFromTuple(tuple)\n- \n- self._db.createTable( self._table, "path" )\n- self._db.createBinPathTable(self._table, True)\n- self._tpA = TableBinPathAdaptator( self._db, self._table )\n- self._tpA.insert(p1)\n- self._tpA.insert(p2)\n- self._tpA.insert(p3)\n- self._tpA.insert(p4)\n- \n- sqlCmd = "SELECT * FROM %s" % ( self._table )\n- self._db.execute( sqlCmd )\n- obsPathTuple = self._db.cursor.fetchall()\n- expPathTuple = ((1, "chr1", 10, 25, "TE1", 11, 17, 1e-18, 20, 87.4),\n- (1, "chr1", 100, 250, "TE1", 17, 11, 1e-18, 20, 87.4),\n- (2, "chr1", 15, 30, "TE2", 10, 13, 5e-24, 34, 93.1),\n- (4, "chr5", 140, 251, "TE5", 140, 251, 2e-14, 14, 73.1),)\n- self.assertEquals(expPathTuple, obsPathTuple)\n-\n- sqlCmd = "SELECT * FROM %s_idx" % ( self._table )\n- self._db.execute( sqlCmd )\n- obsPathTuple = self._db.cursor'..b'uple(tuple)\n- \n- tuple = ("3", "chr1", "15", "30", "TE2", "10", "13", "5e-24", "34", "93.1")\n- p3 = Path()\n- p3.setFromTuple(tuple)\n- \n- self._db.createTable( self._table, "path" )\n- self._db.createBinPathTable(self._table, True)\n- self._tpA = TableBinPathAdaptator( self._db, self._table )\n- self._tpA.insert(p1)\n- self._tpA.insert(p2)\n- self._tpA.insert(p3)\n- \n- expLSet = []\n- obsLSet = self._tpA.getSetListOverlappingQueryCoord(\'chr1\', 5000, 6000)\n- \n- self.assertEquals(expLSet, obsLSet)\n- \n- def test_getSetListOverlappingQueryCoord_one_included_and_two_chain(self):\n- tuple = ("1", "chr1", "10", "25", "TE1", "11", "17", "1e-18", "20", "87.4")\n- p1 = Path()\n- p1.setFromTuple(tuple)\n-\n- tuple = ("2", "chr1", "100", "250", "TE1", "11", "17", "1e-18", "20", "87.4")\n- p2 = Path()\n- p2.setFromTuple(tuple)\n-\n- tuple = ("2", "chr1", "1000", "2500", "TE1", "11", "17", "1e-18", "20", "87.4")\n- p3 = Path()\n- p3.setFromTuple(tuple)\n-\n- tuple = ("3", "chr1", "50", "150", "TE1", "11", "17", "1e-18", "20", "87.4")\n- p4 = Path()\n- p4.setFromTuple(tuple)\n- \n- tuple = ("4", "chr1", "15", "30", "TE2", "10", "13", "5e-24", "34", "93.1")\n- p5 = Path()\n- p5.setFromTuple(tuple)\n- \n- self._db.createTable( self._table, "path" )\n- self._db.createBinPathTable(self._table, True)\n- self._tpA = TableBinPathAdaptator( self._db, self._table )\n- self._tpA.insert(p1)\n- self._tpA.insert(p2)\n- self._tpA.insert(p3)\n- self._tpA.insert(p4)\n- self._tpA.insert(p5)\n- \n- s2 = Set()\n- s2.setFromTuple(("2","TE1","chr1","100","250"))\n- s4 = Set()\n- s4.setFromTuple(("3","TE1","chr1","50","150"))\n- expLSet = [s2, s4]\n- obsLSet = self._tpA.getSetListOverlappingQueryCoord(\'chr1\', 95, 300)\n- \n- self.assertEquals(expLSet, obsLSet)\n- \n- def test_getIdList( self ):\n- p1 = Path()\n- p1.setFromString( "1\\tchr1\\t1\\t10\\tTE1\\t11\\t17\\t1e-20\\t30\\t90.2\\n" )\n- p2 = Path()\n- p2.setFromString( "2\\tchr1\\t2\\t9\\tTE2\\t10\\t13\\t1e-20\\t30\\t90.2\\n" )\n- p3 = Path()\n- p3.setFromString( "2\\tchr1\\t12\\t19\\tTE2\\t15\\t22\\t1e-10\\t40\\t94.2\\n" )\n- p4 = Path()\n- p4.setFromString( "3\\tchr2\\t8\\t13\\tTE1\\t11\\t17\\t1e-20\\t30\\t90.2\\n" )\n- \n- self._db.createTable( self._table, "path" )\n- self._db.createBinPathTable(self._table, True)\n- self._tpA = TableBinPathAdaptator( self._db, self._table )\n- \n- lPath = [ p1, p2, p3, p4]\n- self._tpA.insertList(lPath)\n- \n- expList = [ 1, 2, 3 ]\n- obsList = self._tpA.getIdList()\n- \n- self.assertEqual( expList, obsList )\n- \n- def test_getQueryList(self):\n- tuple = ("1", "chr1", "10", "25", "TE1", "11", "17", "1e-18", "20", "87.4")\n- p1 = Path()\n- p1.setFromTuple(tuple)\n-\n- tuple = ("2", "chr1", "100", "250", "TE1", "11", "17", "1e-18", "20", "87.4")\n- p2 = Path()\n- p2.setFromTuple(tuple)\n- \n- tuple = ("3", "chr2", "15", "30", "TE2", "10", "13", "5e-24", "34", "93.1")\n- p3 = Path()\n- p3.setFromTuple(tuple)\n- \n- self._db.createTable( self._table, "path" )\n- self._db.createBinPathTable(self._table, True)\n- self._tpA = TableBinPathAdaptator( self._db, self._table )\n- self._tpA.insert(p1)\n- self._tpA.insert(p2)\n- self._tpA.insert(p3)\n- \n- expList = [ "chr1", "chr2" ]\n- obsList = self._tpA.getQueryList()\n- self.assertEqual( expList, obsList )\n-\n-test_suite = unittest.TestSuite()\n-test_suite.addTest( unittest.makeSuite( Test_TableBinPathAdaptator ) )\n-if __name__ == \'__main__\':\n- unittest.TextTestRunner(verbosity=2).run( test_suite )\n' |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/test/Test_TableBinSetAdaptator.py --- a/commons/core/sql/test/Test_TableBinSetAdaptator.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| b'@@ -1,290 +0,0 @@\n-import unittest\n-import os\n-import time\n-from commons.core.sql.TableBinSetAdaptator import TableBinSetAdaptator\n-from commons.core.coord.Set import Set\n-from commons.core.sql.DbFactory import DbFactory\n-\n-class Test_TableBinSetAdaptator(unittest.TestCase):\n-\n- def setUp(self):\n- self._uniqId = "%s_%s" % (time.strftime("%Y%m%d%H%M%S") , os.getpid())\n- self._iDb = DbFactory.createInstance()\n- radicalTableName = "dummySetTable"\n- self._tableName = "%s_%s" % (radicalTableName, self._uniqId)\n- self._tableName_bin = "%s_idx" % self._tableName\n- self._setFileName = "dummySetFile_%s" % self._uniqId\n- setF = open( self._setFileName, "w" )\n- setF.write("1\\tseq1\\tchr1\\t1900\\t3900\\n")\n- setF.write("2\\tseq2\\tchr1\\t2\\t9\\n")\n- setF.write("3\\tseq3\\tchr1\\t8\\t13\\n")\n- setF.close()\n- self._iDb.createTable(self._tableName, "set", self._setFileName)\n- self._iTableBinSetAdaptator = TableBinSetAdaptator(self._iDb, self._tableName)\n- \n- def tearDown(self):\n- self._iDb.dropTable( self._tableName )\n- self._iDb.dropTable( self._tableName_bin )\n- self._iDb.close()\n- if os.path.exists(self._setFileName):\n- os.remove(self._setFileName)\n- \n- def test_insASetInSetAndBinTable(self):\n- iSet = Set(1, "set1", "seq1", 2, 1)\n- self._iDb.createBinSetTable(self._tableName, True)\n- self._iTableBinSetAdaptator.insASetInSetAndBinTable(iSet)\n- expTupleInBinTable = ((1L, 10000.0, \'chr1\', 1900L, 3900L, 1L), (2L, 1000.0, \'chr1\', 2L, 9L, 1L), (3L, 1000.0, \'chr1\', 8L, 13L, 1L), (1L, 1000.0, \'seq1\', 1L, 2L, 0L))\n- sqlCmd = "SELECT * FROM %s" % ( self._tableName_bin )\n- self._iDb.execute( sqlCmd )\n- obsTupleInBinTable = self._iDb.cursor.fetchall()\n- self.assertEquals(expTupleInBinTable, obsTupleInBinTable)\n- expTupleInSetTable = ((1L, \'seq1\', \'chr1\', 1900L, 3900L), (2L, \'seq2\', \'chr1\', 2L, 9L), (3L, \'seq3\', \'chr1\', 8L, 13L), (1L, \'set1\', \'seq1\', 2L, 1L))\n- sqlCmd = "SELECT * FROM %s" % ( self._tableName )\n- self._iDb.execute( sqlCmd )\n- obsTupleInSetTable = self._iDb.cursor.fetchall()\n- self.assertEquals(expTupleInSetTable, obsTupleInSetTable)\n- \n- def test_insASetInSetAndBinTable_delayedCase(self):\n- iSet = Set(1, "set1", "seq1", 2, 1)\n- self._iDb.createBinSetTable(self._tableName, True)\n- self._iTableBinSetAdaptator.insASetInSetAndBinTable(iSet, True)\n- expTupleInBinTable = ((1L, 10000.0, \'chr1\', 1900L, 3900L, 1L), (2L, 1000.0, \'chr1\', 2L, 9L, 1L), (3L, 1000.0, \'chr1\', 8L, 13L, 1L), (1L, 1000.0, \'seq1\', 1L, 2L, 0L))\n- sqlCmd = "SELECT * FROM %s" % ( self._tableName_bin )\n- self._iDb.execute( sqlCmd )\n- obsTupleInBinTable = self._iDb.cursor.fetchall()\n- self.assertEquals(expTupleInBinTable, obsTupleInBinTable)\n- expTupleInSetTable = ((1L, \'seq1\', \'chr1\', 1900L, 3900L), (2L, \'seq2\', \'chr1\', 2L, 9L), (3L, \'seq3\', \'chr1\', 8L, 13L), (1L, \'set1\', \'seq1\', 2L, 1L))\n- sqlCmd = "SELECT * FROM %s" % ( self._tableName )\n- self._iDb.execute( sqlCmd )\n- obsTupleInSetTable = self._iDb.cursor.fetchall()\n- self.assertEquals(expTupleInSetTable, obsTupleInSetTable)\n- \n- def test_deleteFromIdFromSetAndBinTable(self):\n- self._iDb.createBinSetTable(self._tableName, True)\n- self._iTableBinSetAdaptator.deleteFromIdFromSetAndBinTable(2)\n- expTupleInBinTable = ((1L, 10000.0, \'chr1\', 1900L, 3900L, 1L), (3L, 1000.0, \'chr1\', 8L, 13L, 1L))\n- sqlCmd = "SELECT * FROM %s" % ( self._tableName_bin )\n- self._iDb.execute( sqlCmd )\n- obsTupleInBinTable = self._iDb.cursor.fetchall()\n- self.assertEquals(expTupleInBinTable, obsTupleInBinTable)\n- expTupleInSetTable = ((1L, \'seq1\', \'chr1\', 1900L, 3900L), (3L, \'seq3\', \'chr1\', 8L, 13L))\n- sqlCmd = "SELECT * FROM %s" % ( self._tableName )\n- '..b' obsTupleInBinTable = self._iDb.cursor.fetchall()\n- self.assertEquals(expTupleInBinTable, obsTupleInBinTable)\n- expTupleInSetTable = ((1L, \'seq1\', \'chr1\', 1900L, 3900L), (5L, \'seq5\', \'chr1\', 1L, 13L), (4L, \'seq4\', \'chr1\', 100L, 390L) )\n- sqlCmd = "SELECT * FROM %s" % ( self._tableName )\n- self._iDb.execute( sqlCmd )\n- obsTupleInSetTable = self._iDb.cursor.fetchall()\n- self.assertEquals(expTupleInSetTable, obsTupleInSetTable)\n-\n- def test_insertListInSetAndBinTableAndRemoveOverlaps(self):\n- iSet1 = Set(1, "seq4", "chr1", 100, 390)\n- iSet2 = Set(2, "seq5", "chr1", 1, 13)\n- lSet = [iSet1, iSet2]\n- self._iDb.createBinSetTable(self._tableName, True)\n- self._iTableBinSetAdaptator.insertListInSetAndBinTableAndRemoveOverlaps(lSet)\n- expTupleInBinTable = ((1L, 10000.0, \'chr1\', 1900L, 3900L, 1L), (2L, 1000.0, \'chr1\', 2L, 9L, 1L), (3L, 1000.0, \'chr1\', 8L, 13L, 1L), (4L, 1000.0, \'chr1\', 100L, 390L, 1L))\n- sqlCmd = "SELECT * FROM %s" % ( self._tableName_bin )\n- self._iDb.execute( sqlCmd )\n- obsTupleInBinTable = self._iDb.cursor.fetchall()\n- self.assertEquals(expTupleInBinTable, obsTupleInBinTable)\n- expTupleInSetTable = ((1L, \'seq1\', \'chr1\', 1900L, 3900L), (2L, \'seq2\', \'chr1\', 2L, 9L), (3L, \'seq3\', \'chr1\', 8L, 13L), (4L, \'seq4\', \'chr1\', 100L, 390L))\n- sqlCmd = "SELECT * FROM %s" % ( self._tableName )\n- self._iDb.execute( sqlCmd )\n- obsTupleInSetTable = self._iDb.cursor.fetchall()\n- self.assertEquals(expTupleInSetTable, obsTupleInSetTable)\n-\n- def test_insertListInSetAndBinTableAndRemoveOverlaps_Without_Overlaps(self):\n- iSet1 = Set(1, "seq4", "chr1", 100, 390)\n- iSet2 = Set(2, "seq5", "chr1", 50, 65)\n- lSet = [iSet1, iSet2]\n- self._iDb.createBinSetTable(self._tableName, True)\n- self._iTableBinSetAdaptator.insertListInSetAndBinTableAndRemoveOverlaps(lSet)\n- expTupleInBinTable = ((1L, 10000.0, \'chr1\', 1900L, 3900L, 1L), (2L, 1000.0, \'chr1\', 2L, 9L, 1L), (3L, 1000.0, \'chr1\', 8L, 13L, 1L), (4L, 1000.0, \'chr1\', 100L, 390L, 1L), (5L, 1000.0, \'chr1\', 50L, 65L, 1L))\n- sqlCmd = "SELECT * FROM %s" % ( self._tableName_bin )\n- self._iDb.execute( sqlCmd )\n- obsTupleInBinTable = self._iDb.cursor.fetchall()\n- self.assertEquals(expTupleInBinTable, obsTupleInBinTable)\n- expTupleInSetTable = ((1L, \'seq1\', \'chr1\', 1900L, 3900L), (2L, \'seq2\', \'chr1\', 2L, 9L), (3L, \'seq3\', \'chr1\', 8L, 13L), (4L, \'seq4\', \'chr1\', 100L, 390L), (5L, \'seq5\', \'chr1\', 50L, 65L))\n- sqlCmd = "SELECT * FROM %s" % ( self._tableName )\n- self._iDb.execute( sqlCmd )\n- obsTupleInSetTable = self._iDb.cursor.fetchall()\n- self.assertEquals(expTupleInSetTable, obsTupleInSetTable)\n-\n- def test_insertListInSetAndBinTableAndRemoveOverlaps_With_Only_Overlaps(self):\n- iSet1 = Set(1, "seq4", "chr1", 1, 5)\n- iSet2 = Set(2, "seq5", "chr1", 8, 13)\n- lSet = [iSet1, iSet2]\n- self._iDb.createBinSetTable(self._tableName, True)\n- self._iTableBinSetAdaptator.insertListInSetAndBinTableAndRemoveOverlaps(lSet)\n- expTupleInBinTable = ((1L, 10000.0, \'chr1\', 1900L, 3900L, 1L), (2L, 1000.0, \'chr1\', 2L, 9L, 1L), (3L, 1000.0, \'chr1\', 8L, 13L, 1L))\n- sqlCmd = "SELECT * FROM %s" % ( self._tableName_bin )\n- self._iDb.execute( sqlCmd )\n- obsTupleInBinTable = self._iDb.cursor.fetchall()\n- self.assertEquals(expTupleInBinTable, obsTupleInBinTable)\n- expTupleInSetTable = ((1L, \'seq1\', \'chr1\', 1900L, 3900L), (2L, \'seq2\', \'chr1\', 2L, 9L), (3L, \'seq3\', \'chr1\', 8L, 13L))\n- sqlCmd = "SELECT * FROM %s" % ( self._tableName )\n- self._iDb.execute( sqlCmd )\n- obsTupleInSetTable = self._iDb.cursor.fetchall()\n- self.assertEquals(expTupleInSetTable, obsTupleInSetTable)\n- \n-if __name__ == "__main__":\n- unittest.main()\n\\ No newline at end of file\n' |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/test/Test_TableJobAdaptator.py --- a/commons/core/sql/test/Test_TableJobAdaptator.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| b'@@ -1,640 +0,0 @@\n-import unittest\n-import sys\n-import os\n-import time\n-#import stat\n-#import threading\n-from commons.core.sql.DbMySql import DbMySql\n-#from commons.core.sql.DbSQLite import DbSQLite\n-from commons.core.sql.Job import Job\n-from commons.core.utils.FileUtils import FileUtils\n-from commons.core.sql.TableJobAdaptatorFactory import TableJobAdaptatorFactory\n-\n-#class Test_TableJobAdaptator_SQLite( unittest.TestCase ):\n-# \n-# def setUp(self):\n-# self._jobTableName = "dummyJobTable"\n-# self._dbName = "test.db"\n-# self._db = DbSQLite(self._dbName)\n-# self._iTJA = TableJobAdaptator(self._db, self._jobTableName)\n-# if not self._db.doesTableExist(self._jobTableName):\n-# self._db.createJobTable(self._jobTableName)\n-# self._iJob = self._createJobInstance()\n-# \n-# def tearDown(self):\n-# self._iTJA = None\n-# self._db.close()\n-## self._db.delete()\n-# \n-## def test_recordJob(self):\n-## self._iTJA.recordJob(self._iJob)\n-## qryParams = "SELECT jobid, groupid, command, launcher, queue, status, node FROM " + self._jobTableName + " WHERE jobid = ?" \n-## params = (self._iJob.jobid,)\n-## self._db.execute(qryParams, params)\n-## tObs = self._db.fetchall()[0]\n-## tExp =(self._iJob.jobid, self._iJob.groupid, self._iJob.command, self._iJob.launcher, self._iJob.queue, "waiting", "?")\n-## self.assertEquals(tExp,tObs)\n-## \n-## def test_removeJob(self):\n-## self._iTJA.recordJob(self._iJob)\n-## self._iTJA.removeJob(self._iJob)\n-## self.assertTrue(self._db.isEmpty(self._jobTableName))\n-## \n-## def test_getJobStatus(self):\n-## self._iTJA.recordJob(self._iJob)\n-## expStatus = "waiting"\n-## obsStatus = self._iTJA.getJobStatus(self._iJob)\n-## self.assertEquals(expStatus, obsStatus)\n-## \n-## def test_getJobStatus_no_job(self):\n-## expStatus = "unknown"\n-## obsStatus = self._iTJA.getJobStatus(self._iJob)\n-## self.assertEquals(expStatus, obsStatus)\n-##\n-## def test_getJobStatus_no_name(self):\n-## iJob = Job( self._jobTableName, 20, "", "groupid", "queue", "command", "launcherFile", "node", "lResources" ) \n-## expStatus = "unknown"\n-## obsStatus = self._iTJA.getJobStatus(iJob)\n-## self.assertEquals(expStatus, obsStatus)\n-## \n-## def test_getJobStatus_two_jobs(self):\n-## # Warning : this case will not append, because recordJob() begin by removeJob()\n-## sqlCmd = "INSERT INTO %s" % self._iJob.tablename\n-## sqlCmd += " VALUES ("\n-## sqlCmd += " \\"%s\\"," % self._iJob.jobid\n-## sqlCmd += " \\"%s\\"," % self._iJob.jobname\n-## sqlCmd += " \\"%s\\"," % self._iJob.groupid\n-## sqlCmd += " \\"%s\\"," % self._iJob.command.replace("\\"","\\\'")\n-## sqlCmd += " \\"%s\\"," % self._iJob.launcher\n-## sqlCmd += " \\"%s\\"," % self._iJob.queue\n-## sqlCmd += " \\"waiting\\","\n-## sqlCmd += " \\"%s\\"," % time.strftime( "%Y-%m-%d %H:%M:%S" )\n-## sqlCmd += " \\"?\\" );"\n-## self._db.execute(sqlCmd)\n-## self._db.execute(sqlCmd)\n-## \n-## expError = "expError.txt"\n-## expErrorHandler = open(expError, "w")\n-## expErrorHandler.write("ERROR while getting job status: non-unique jobs\\n")\n-## expErrorHandler.close()\n-## obsError = "obsError.txt"\n-## obsErrorHandler = open(obsError, "w")\n-## stderrRef = sys.stderr\n-## sys.stderr = obsErrorHandler\n-## \n-## isSysExitRaised = False\n-## try:\n-## self._iTJA.getJobStatus(self._iJob)\n-## except SystemExit:\n-## isSysExitRaised = True\n-## \n-## obsErrorHandler.close()\n-## \n-## self.assertTrue(isSysExitRaised)\n-## self.assertTrue(FileUtils.are2FilesIdentical(expError, obsError))\n-## sys.stderr = stderrRef\n-## os.remove(obs'..b':\n- obs = False\n- self._iTJA.recordJob(self._iJob)\n- self._iTJA.changeJobStatus(self._iJob, "error")\n- try:\n- self._iTJA.waitJobGroup(self._iJob.groupid, 0, 0)\n- except SystemExit:\n- obs = True\n- self.assertTrue(obs)\n- \n- #TODO: how to test ?!?\n-# def test_waitJobGroup_with_error_relaunch(self):\n-# iJob = Job(0, "job1", "groupid", "queue.q", "command", "launcherFile", "node", ["mem_free=10M", "test=TRUE"])\n-# obs = False\n-# self._iTJA.recordJob(iJob)\n-# self._iTJA.changeJobStatus(iJob, "error")\n-# try:\n-# self._iTJA.waitJobGroup(iJob.groupid)\n-# except SystemExit:\n-# obs = True\n-# self.assertTrue(obs)\n- \n- def test_updateJobIdInDB(self):\n- self._iTJA.recordJob(self._iJob)\n- self._iTJA.updateJobIdInDB(self._iJob, 1000)\n- qryParams = "SELECT jobid FROM " + self._jobTableName + " WHERE jobname = %s AND queue = %s AND groupid = %s" \n- params = (self._iJob.jobname, self._iJob.queue, self._iJob.groupid)\n- self._db.execute(qryParams, params)\n- tObs = self._db.fetchall()[0]\n- tExp =(1000,)\n- self.assertEquals(tExp,tObs)\n-\n- def test_getNodesListByGroupId(self):\n- iJob1 = Job(0, "job1", "groupid", "queue", "command", "launcherFile", "node1", "lResources")\n- iJob2 = Job(1, "job2", "groupid", "queue", "command", "launcherFile", "node2", "lResources")\n- iJob3 = Job(2, "job3", "groupid", "queue", "command", "launcherFile", "node2", "lResources")\n- iJob4 = Job(3, "job4", "groupid2", "queue", "command", "launcherFile", "node3", "lResources")\n- self._insertJob(iJob1)\n- self._insertJob(iJob2)\n- self._insertJob(iJob3)\n- self._insertJob(iJob4)\n- expNodeList = ["node1", "node2"]\n- obsNodeList = self._iTJA.getNodesListByGroupId("groupid")\n- self.assertEquals(expNodeList, obsNodeList)\n-\n- def test_getNodesListByGroupId_empty_list(self):\n- iJob1 = Job(0, "job1", "groupid", "queue", "command", "launcherFile", "node1", "lResources")\n- iJob2 = Job(1, "job2", "groupid", "queue", "command", "launcherFile", "node2", "lResources")\n- iJob3 = Job(2, "job3", "groupid32", "queue", "command", "launcherFile", "node3", "lResources")\n- self._insertJob(iJob1)\n- self._insertJob(iJob2)\n- self._insertJob(iJob3)\n- expNodeList = []\n- obsNodeList = self._iTJA.getNodesListByGroupId("groupid3")\n- self.assertEquals(expNodeList, obsNodeList)\n- \n-# TODO test TableJobAdaptator._createJobInstance TableJobAdaptator._createLauncherFile\n- def _insertJob(self, iJob):\n- self._iTJA = TableJobAdaptatorFactory.createInstance(self._db, self._jobTableName) \n- self._iTJA.removeJob(iJob)\n- sqlCmd = "INSERT INTO %s" % self._jobTableName\n- sqlCmd += " VALUES ("\n- sqlCmd += " \\"%s\\"," % iJob.jobid\n- sqlCmd += " \\"%s\\"," % iJob.jobname\n- sqlCmd += " \\"%s\\"," % iJob.groupid\n- sqlCmd += " \\"%s\\"," % iJob.launcher\n- sqlCmd += " \\"%s\\"," % iJob.queue\n- sqlCmd += " \\"%s\\"," % iJob.lResources\n- sqlCmd += " \\"waiting\\","\n- sqlCmd += " \\"%s\\"," % time.strftime("%Y-%m-%d %H:%M:%S")\n- sqlCmd += " \\"%s\\" );" % iJob.node\n- self._db.execute(sqlCmd)\n-\n- def _createJobInstance(self):\n- return Job(0, "job1", "groupid", "", "command", "launcherFile", "node", ["mem_free=10M"])\n-\n-#class RecordJobThread(threading.Thread):\n-#\n-# def __init__(self, iTableJobAdaptator, iJob):\n-# threading.Thread.__init__(self)\n-# self._iTableJobAdaptator = iTableJobAdaptator\n-# self._iJob = iJob\n-# \n-# def run(self):\n-# self._iTableJobAdaptator.recordJob(self._iJob)\n-# #self._iTableJobAdaptator.submitJob(self._iJob)\n- \n-if __name__ == "__main__":\n- unittest.main()\n' |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/test/Test_TableJobAdaptatorFactory.py --- a/commons/core/sql/test/Test_TableJobAdaptatorFactory.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| @@ -1,27 +0,0 @@ -import os -import unittest -from commons.core.sql.TableJobAdaptatorFactory import TableJobAdaptatorFactory -from commons.core.sql.DbFactory import DbFactory - -class Test_TableJobAdaptatorFactory(unittest.TestCase): - - def test_createInstance_SGE(self): - REPET_JOB_MANAGER_Initial_Value = os.environ["REPET_JOB_MANAGER"] - os.environ["REPET_JOB_MANAGER"] = "SGE" - instance = TableJobAdaptatorFactory.createInstance(DbFactory.createInstance(), "dummyJobTable") - obsClassName = instance.__class__.__name__ - expClassName = "TableJobAdaptatorSGE" - os.environ["REPET_JOB_MANAGER"] = REPET_JOB_MANAGER_Initial_Value - self.assertEquals(expClassName, obsClassName) - - def test_createInstance_Torque(self): - REPET_JOB_MANAGER_Initial_Value = os.environ["REPET_JOB_MANAGER"] - os.environ["REPET_JOB_MANAGER"] = "Torque" - instance = TableJobAdaptatorFactory.createInstance(DbFactory.createInstance(), "dummyJobTable") - obsClassName = instance.__class__.__name__ - expClassName = "TableJobAdaptatorTorque" - os.environ["REPET_JOB_MANAGER"] = REPET_JOB_MANAGER_Initial_Value - self.assertEquals(expClassName, obsClassName) - -if __name__ == "__main__": - unittest.main() \ No newline at end of file |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/test/Test_TableMapAdaptator.py --- a/commons/core/sql/test/Test_TableMapAdaptator.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| b'@@ -1,250 +0,0 @@\n-# Copyright INRA (Institut National de la Recherche Agronomique)\n-# http://www.inra.fr\n-# http://urgi.versailles.inra.fr\n-#\n-# This software is governed by the CeCILL license under French law and\n-# abiding by the rules of distribution of free software. You can use, \n-# modify and/ or redistribute the software under the terms of the CeCILL\n-# license as circulated by CEA, CNRS and INRIA at the following URL\n-# "http://www.cecill.info". \n-#\n-# As a counterpart to the access to the source code and rights to copy,\n-# modify and redistribute granted by the license, users are provided only\n-# with a limited warranty and the software\'s author, the holder of the\n-# economic rights, and the successive licensors have only limited\n-# liability. \n-#\n-# In this respect, the user\'s attention is drawn to the risks associated\n-# with loading, using, modifying and/or developing or reproducing the\n-# software by the user in light of its specific status of free software,\n-# that may mean that it is complicated to manipulate, and that also\n-# therefore means that it is reserved for developers and experienced\n-# professionals having in-depth computer knowledge. Users are therefore\n-# encouraged to load and test the software\'s suitability as regards their\n-# requirements in conditions enabling the security of their systems and/or \n-# data to be ensured and, more generally, to use and operate it in the \n-# same conditions as regards security. \n-#\n-# The fact that you are presently reading this means that you have had\n-# knowledge of the CeCILL license and that you accept its terms.\n-\n-\n-import unittest\n-import time\n-import os\n-from commons.core.sql.TableMapAdaptator import TableMapAdaptator\n-from commons.core.sql.DbMySql import DbMySql\n-from commons.core.coord.Map import Map\n-from commons.core.coord.Set import Set\n-\n-\n-class Test_TableMapAdaptator( unittest.TestCase ):\n- \n- def setUp( self ):\n- self._uniqId = "%s_%s" % ( time.strftime("%Y%m%d%H%M%S") , os.getpid() )\n- self._configFileName = "dummyConfigFile_%s" % ( self._uniqId )\n- configF = open(self._configFileName, "w" )\n- configF.write( "[repet_env]\\n" )\n- configF.write( "repet_host: %s\\n" % ( os.environ["REPET_HOST"] ) )\n- configF.write( "repet_user: %s\\n" % ( os.environ["REPET_USER"] ) )\n- configF.write( "repet_pw: %s\\n" % ( os.environ["REPET_PW"] ) )\n- configF.write( "repet_db: %s\\n" % ( os.environ["REPET_DB"] ) )\n- configF.write( "repet_port: %s\\n" % ( os.environ["REPET_PORT"] ) )\n- configF.close()\n- self._iDb = DbMySql( cfgFileName=self._configFileName )\n- self._table = "dummyMapTable_%s" % ( self._uniqId )\n- self._tMapA = TableMapAdaptator( self._iDb, self._table )\n- \n- \n- def tearDown( self ):\n- self._uniqId = None\n- self._iDb.dropTable( self._table )\n- self._iDb.close()\n- self._table = None\n- self._tMapA = None\n- os.remove( self._configFileName )\n- self._configFileName = ""\n- \n-##################################################################################\n-################## Tests for methods in ITableMapAdaptator #######################\n-################################################################################## \n-\n- def test_getEndFromSeqName(self):\n- self._iDb.createTable( self._table, "map", "" )\n- map1 = Map()\n- map1.setFromString( "name1\\tdesc1\\t1\\t120\\n" )\n- map2 = Map()\n- map2.setFromString( "name2\\tdesc2\\t1\\t20\\n" )\n- for m in [ map1, map2]:\n- self._tMapA.insert(m)\n- expEnd = 20\n- obsEnd = self._tMapA.getEndFromSeqName("desc2")\n- self.assertEqual(expEnd, obsEnd) \n- \n-\n- def test_getMapListFromSeqName( self ):\n- self._iDb.createTable( self._table, "map", "" )\n- map1 = Map()\n- map1.setFromString( "name1\\tdesc1\\t1\\t120\\n" )\n- map2 = Map()\n- '..b' map1.setFromString( "name1\\tdesc1\\t1\\t120\\n" )\n- map2 = Map()\n- map2.setFromString( "name2\\tdesc2\\t1\\t20\\n" )\n- map3 = Map()\n- map3.setFromString( "name2\\tdesc2\\t1\\t50\\n" )\n- for m in [ map1, map2, map3 ]: self._tMapA.insert( m )\n- explMap = [Set( 1,"name2", "desc2", 1, 20), Set( 2,"name2", "desc2", 1, 50)]\n- obslMap = self._tMapA.getSetListFromSeqName("name2")\n- self.assertEqual( explMap, obslMap )\n- \n- def test_getMapListOverlappingCoord( self ):\n- self._iDb.createTable( self._table, "map", "" )\n- map1 = Map()\n- map1.setFromString( "name1\\tdesc1\\t70\\t120\\n" )\n- map2 = Map()\n- map2.setFromString( "name2\\tdesc1\\t1\\t20\\n" )\n- map3 = Map()\n- map3.setFromString( "name3\\tdesc1\\t1\\t50\\n" ) \n- for m in [ map1, map2, map3 ]: self._tMapA.insert( m )\n- explMap = [Map("name2", "desc1", 1, 20), Map("name3", "desc1", 1, 50)]\n- obslMap = self._tMapA.getMapListOverlappingCoord("desc1", 1, 60)\n- self.assertEqual( explMap, obslMap )\n- \n- def test_getSetListOverlappingCoord( self ):\n- self._iDb.createTable( self._table, "map", "" )\n- map1 = Map()\n- map1.setFromString( "name1\\tdesc1\\t70\\t120\\n" )\n- map2 = Map()\n- map2.setFromString( "name2\\tdesc1\\t1\\t20\\n" )\n- map3 = Map()\n- map3.setFromString( "name3\\tdesc1\\t1\\t50\\n" ) \n- for m in [ map1, map2, map3 ]: self._tMapA.insert( m )\n- explSet = [Set(1, "name2", "desc1", 1, 20), Set(2, "name3", "desc1", 1, 50)]\n- obslSet = self._tMapA.getSetListOverlappingCoord("desc1", 1, 60)\n- self.assertEqual( explSet, obslSet )\n- \n-##################################################################################\n-########################### Tests for other methods ##############################\n-##################################################################################\n- \n- def test_getListOfAllMaps( self ):\n- self._iDb.createTable( self._table, "map", "" )\n- map1 = Map()\n- map1.setFromString( "name1\\tdesc1\\t1\\t120\\n" )\n- map2 = Map()\n- map2.setFromString( "name2\\tdesc2\\t1\\t20\\n" )\n- for m in [ map1, map2 ]: self._tMapA.insert( m )\n- lExp = [ map1, map2 ]\n- lObs = self._tMapA.getListOfAllMaps()\n- self.assertEqual( lObs, lExp )\n- \n- def test_getDictPerNameFromMapFile( self ):\n- self._iDb.createTable( self._table, "map", "" )\n- iMap1 = Map( "chunk1", "chromosome1", 1, 100 )\n- iMap2 = Map( "chunk2", "chromosome1", 91, 190 )\n- iMap3 = Map( "chunk3", "chromosome2", 1, 100 )\n- iMap4 = Map( "chunk1", "chromosome1", 1, 100 ) # redundant with iMap1\n- for iMap in [ iMap1, iMap2, iMap3, iMap4 ]:\n- self._tMapA.insert( iMap )\n- dExp = { "chunk1": iMap1, "chunk2": iMap2, "chunk3": iMap3 }\n- dObs = self._tMapA.getDictPerName()\n- self.assertEquals( dExp, dObs )\n- \n-#TODO: Check getListFromSeqName method: uses name instead of seqname\n-# def test_getMapListFromSeqNameList( self ):\n-# self._iDb.createTable( self._table, "map", "" )\n-# map1 = Map()\n-# map1.setFromString( "name1\\tdesc1\\t1\\t120\\n" )\n-# map2 = Map()\n-# map2.setFromString( "name2\\tdesc2\\t1\\t20\\n" )\n-# map3 = Map()\n-# map3.setFromString( "name3\\tdesc2\\t1\\t10\\n" )\n-# map4 = Map()\n-# map4.setFromString( "name4\\tdesc3\\t10\\t200\\n" )\n-# for m in [map1, map2, map3, map4]: self._tMapA.insert( m )\n-# \n-# lMapToRetrieve = ["name1", "desc2"]\n-# lExp = [map1, map2, map3]\n-# lObs = self._tMapA.getMapListFromSeqNameList(lMapToRetrieve)\n-# self.assertEqual( lObs, lExp )\n- \n-test_suite = unittest.TestSuite()\n-test_suite.addTest( unittest.makeSuite( Test_TableMapAdaptator ) )\n-if __name__ == "__main__":\n- unittest.TextTestRunner(verbosity=2).run( test_suite )\n' |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/test/Test_TableMatchAdaptator.py --- a/commons/core/sql/test/Test_TableMatchAdaptator.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| b'@@ -1,264 +0,0 @@\n-# Copyright INRA (Institut National de la Recherche Agronomique)\n-# http://www.inra.fr\n-# http://urgi.versailles.inra.fr\n-#\n-# This software is governed by the CeCILL license under French law and\n-# abiding by the rules of distribution of free software. You can use, \n-# modify and/ or redistribute the software under the terms of the CeCILL\n-# license as circulated by CEA, CNRS and INRIA at the following URL\n-# "http://www.cecill.info". \n-#\n-# As a counterpart to the access to the source code and rights to copy,\n-# modify and redistribute granted by the license, users are provided only\n-# with a limited warranty and the software\'s author, the holder of the\n-# economic rights, and the successive licensors have only limited\n-# liability. \n-#\n-# In this respect, the user\'s attention is drawn to the risks associated\n-# with loading, using, modifying and/or developing or reproducing the\n-# software by the user in light of its specific status of free software,\n-# that may mean that it is complicated to manipulate, and that also\n-# therefore means that it is reserved for developers and experienced\n-# professionals having in-depth computer knowledge. Users are therefore\n-# encouraged to load and test the software\'s suitability as regards their\n-# requirements in conditions enabling the security of their systems and/or \n-# data to be ensured and, more generally, to use and operate it in the \n-# same conditions as regards security. \n-#\n-# The fact that you are presently reading this means that you have had\n-# knowledge of the CeCILL license and that you accept its terms.\n-\n-\n-import unittest\n-import time\n-import os\n-from commons.core.sql.DbMySql import DbMySql\n-from commons.core.coord.Match import Match\n-from commons.core.sql.TableMatchAdaptator import TableMatchAdaptator\n-\n-\n-class Test_TableMatchAdaptator( unittest.TestCase ):\n- \n- def setUp( self ):\n- self._uniqId = "%s_%s" % (time.strftime("%Y%m%d%H%M%S") , os.getpid())\n- self._configFileName = "dummyConfigFile_%s" % self._uniqId\n- self._iDb = DbMySql()\n- self._table = "dummyMatchTable_%s" % self._uniqId\n- self._tMatchA = TableMatchAdaptator( self._iDb, self._table )\n- \n- def tearDown( self ):\n- self._uniqId = None\n- self._iDb.dropTable( self._table )\n- self._iDb.close()\n- self._table = None\n- self._tMatchA = None\n- \n-##################################################################################\n-################## Tests for methods in ITableMatchAdaptator #####################\n-################################################################################## \n- def test_insert(self):\n- match = Match() \n-\n- tuple = ("QName1", 1, 5, 5, 0.1, 0.2, "SName1", 5, 25, 20, 0.15, 1e-20, 15, 87.2, 1)\n- \n- match.setFromTuple(tuple)\n- \n- self._iDb.createTable( self._table, "match", "" ) \n- self._tMatchA.insert( match, False )\n- \n- expTMatchTuple = ((\'QName1\', 1L, 5L, 5L, 0.1, 0.2, \'SName1\', 5L, 25L, 20L, 0.15, 1e-20, 15L, 87.2, 1L),)\n- \n- sqlCmd = "SELECT * FROM %s" % ( self._table )\n- self._iDb.execute( sqlCmd )\n- obsTmatchTuple = self._iDb.cursor.fetchall()\n- \n- self.assertEquals( expTMatchTuple, obsTmatchTuple )\n- \n-\n- def test_insert_empty_match(self):\n- match = Match() \n-\n- tuple = ("", -1, -1, 5, 0.1, 0.2, "SName1", 5, 25, 20, 0.15, 1e-20, 15, 87.2, 1)\n- \n- match.setFromTuple(tuple)\n- \n- self._iDb.createTable( self._table, "match", "" ) \n- self._tMatchA.insert( match, False )\n- \n- expTMatchTuple = ()\n- \n- sqlCmd = "SELECT * FROM %s" % ( self._table )\n- self._iDb.execute( sqlCmd )\n- obsTmatchTuple = self._iDb.cursor.fetchall()\n- \n- self.assertEquals( expTMatchTuple, obsTmatchTuple'..b' = Match()\n- match1.setFromTuple( tuple1 )\n- match2 = Match()\n- match2.setFromTuple( tuple2 )\n- match3 = Match()\n- match3.setFromTuple( tuple3 )\n- match4 = Match()\n- match4.setFromTuple( tuple4 )\n- lMatch = [ match1, match2, match3, match4 ]\n- expListMatch = [ match1 ]\n- self._tMatchA.insertList(lMatch)\n- \n- obsListMatch = self._tMatchA.getMatchListFromId(1)\n- \n- self.assertEquals(expListMatch, obsListMatch)\n- \n- \n- def test_getMatchListFromIdList_empty_id_list( self ):\n- self._iDb.createTable( self._table, "match", "" )\n- tuple1 = ("QName", 1, 5, 5, 0.1, 0.2, "SName", 5, 25, 20, 0.15, 1e-20, 15, 87.2, 1)\n- tuple2 = ("QName", 1, 6, 6, 0.2, 0.1, "SName", 6, 26, 10, 0.18, 1e-30, 18, 85.2, 2)\n- tuple3 = ("QName", 1, 7, 8, 0.1, 0.2, "SName", 5, 20, 15, 0.20, 1e-25, 20, 89.0, 3)\n- tuple4 = ("QName", 1, 8, 8, 0.1, 0.1, "SName", 5, 15, 10, 0.17, 1e-23, 14, 89.5, 4)\n- match1 = Match()\n- match1.setFromTuple( tuple1 )\n- match2 = Match()\n- match2.setFromTuple( tuple2 )\n- match3 = Match()\n- match3.setFromTuple( tuple3 )\n- match4 = Match()\n- match4.setFromTuple( tuple4 )\n- lMatch = [ match1, match2, match3, match4 ]\n- self._tMatchA.insertList(lMatch)\n- \n- expList = []\n- obsList = self._tMatchA.getMatchListFromIdList([])\n- self.assertEquals(expList, obsList)\n- \n- \n- def test_getMatchListFromIdList( self ):\n- self._iDb.createTable( self._table, "match", "" )\n- tuple1 = ("QName", 1, 5, 5, 0.1, 0.2, "SName", 5, 25, 20, 0.15, 1e-20, 15, 87.2, 1)\n- tuple2 = ("QName", 1, 6, 6, 0.2, 0.1, "SName", 6, 26, 10, 0.18, 1e-30, 18, 85.2, 2)\n- tuple3 = ("QName", 1, 7, 8, 0.1, 0.2, "SName", 5, 20, 15, 0.20, 1e-25, 20, 89.0, 3)\n- tuple4 = ("QName", 1, 8, 8, 0.1, 0.1, "SName", 5, 15, 10, 0.17, 1e-23, 14, 89.5, 4)\n- match1 = Match()\n- match1.setFromTuple( tuple1 )\n- match2 = Match()\n- match2.setFromTuple( tuple2 )\n- match3 = Match()\n- match3.setFromTuple( tuple3 )\n- match4 = Match()\n- match4.setFromTuple( tuple4 )\n- lMatch = [ match1, match2, match3, match4 ]\n- self._tMatchA.insertList(lMatch)\n- \n- lObs = self._tMatchA.getMatchListFromIdList((1, 2, 3))\n- \n- lExp = [match1, match2, match3]\n- self.assertEquals(lExp, lObs)\n- \n- def test_getListOfAllMatches( self ):\n- self._iDb.createTable( self._table, "match", "" )\n- tuple1 = ("QName", 1, 5, 5, 0.1, 0.2, "SName", 5, 25, 20, 0.15, 1e-20, 15, 87.2, 1)\n- tuple2 = ("QName", 1, 6, 6, 0.2, 0.1, "SName", 6, 26, 10, 0.18, 1e-30, 18, 85.2, 2)\n- tuple3 = ("QName", 1, 7, 8, 0.1, 0.2, "SName", 5, 20, 15, 0.20, 1e-25, 20, 89.0, 3)\n- tuple4 = ("QName", 1, 8, 8, 0.1, 0.1, "SName", 5, 15, 10, 0.17, 1e-23, 14, 89.5, 4)\n- match1 = Match()\n- match1.setFromTuple( tuple1 )\n- match2 = Match()\n- match2.setFromTuple( tuple2 )\n- match3 = Match()\n- match3.setFromTuple( tuple3 )\n- match4 = Match()\n- match4.setFromTuple( tuple4 )\n- lMatch = [ match1, match2, match3, match4 ]\n- expList = [ match1, match2, match3, match4 ]\n- self._tMatchA.insertList(lMatch)\n-\n- obsList = self._tMatchA.getListOfAllMatches()\n- self.assertEqual( expList, obsList )\n- \n- \n- def test_getListOfAllMatches_empty_table( self ):\n- self._iDb.createTable( self._table, "match", "" )\n- expList = []\n- obsList = self._tMatchA.getListOfAllMatches()\n- self.assertEqual( expList, obsList )\n- \n- \n-test_suite = unittest.TestSuite()\n-test_suite.addTest( unittest.makeSuite( Test_TableMatchAdaptator ) )\n-if __name__ == "__main__":\n- unittest.TextTestRunner(verbosity=2).run( test_suite )\n' |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/test/Test_TablePathAdaptator.py --- a/commons/core/sql/test/Test_TablePathAdaptator.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| b'@@ -1,1376 +0,0 @@\n-# Copyright INRA (Institut National de la Recherche Agronomique)\n-# http://www.inra.fr\n-# http://urgi.versailles.inra.fr\n-#\n-# This software is governed by the CeCILL license under French law and\n-# abiding by the rules of distribution of free software. You can use, \n-# modify and/ or redistribute the software under the terms of the CeCILL\n-# license as circulated by CEA, CNRS and INRIA at the following URL\n-# "http://www.cecill.info". \n-#\n-# As a counterpart to the access to the source code and rights to copy,\n-# modify and redistribute granted by the license, users are provided only\n-# with a limited warranty and the software\'s author, the holder of the\n-# economic rights, and the successive licensors have only limited\n-# liability. \n-#\n-# In this respect, the user\'s attention is drawn to the risks associated\n-# with loading, using, modifying and/or developing or reproducing the\n-# software by the user in light of its specific status of free software,\n-# that may mean that it is complicated to manipulate, and that also\n-# therefore means that it is reserved for developers and experienced\n-# professionals having in-depth computer knowledge. Users are therefore\n-# encouraged to load and test the software\'s suitability as regards their\n-# requirements in conditions enabling the security of their systems and/or \n-# data to be ensured and, more generally, to use and operate it in the \n-# same conditions as regards security. \n-#\n-# The fact that you are presently reading this means that you have had\n-# knowledge of the CeCILL license and that you accept its terms.\n-\n-\n-import unittest\n-import os\n-import time\n-from commons.core.sql.TablePathAdaptator import TablePathAdaptator\n-from commons.core.coord.Path import Path\n-from commons.core.coord.Set import Set\n-from commons.core.sql.DbMySql import DbMySql\n-from commons.core.coord.Range import Range\n-from commons.core.coord.PathUtils import PathUtils\n-from copy import deepcopy\n-\n-class Test_TablePathAdaptator( unittest.TestCase ):\n- \n- def setUp( self ):\n- self._uniqId = "%s_%s" % ( time.strftime("%Y%m%d%H%M%S") , os.getpid() )\n- self._configFileName = "dummyConfigFile_%s" % ( self._uniqId )\n- configF = open(self._configFileName, "w" )\n- configF.write( "[repet_env]\\n" )\n- configF.write( "repet_host: %s\\n" % ( os.environ["REPET_HOST"] ) )\n- configF.write( "repet_user: %s\\n" % ( os.environ["REPET_USER"] ) )\n- configF.write( "repet_pw: %s\\n" % ( os.environ["REPET_PW"] ) )\n- configF.write( "repet_db: %s\\n" % ( os.environ["REPET_DB"] ) )\n- configF.write( "repet_port: %s\\n" % ( os.environ["REPET_PORT"] ) )\n- configF.close()\n- self._db = DbMySql( cfgFileName = self._configFileName )\n- self._table = "dummyPathTable_%s" % ( self._uniqId )\n- self._tpA = TablePathAdaptator( self._db, self._table )\n- \n- \n- def tearDown( self ):\n- self._uniqId = None\n- self._db.dropTable( self._table )\n- self._db.close()\n- self._table = None\n- self._tMatchA = None\n- os.remove( self._configFileName )\n- self._configFileName = "" \n- \n- \n-##################################################################################\n-################## Tests for methods in ITableMapAdaptator #######################\n-################################################################################## \n- \n- def test_getPathListFromId( self ):\n- pathFileName = "dummyPathFile_%s" % ( self._uniqId )\n- pathF = open( pathFileName, "w" )\n- pathF.write( "1\\tchr1\\t1\\t6\\tTE2\\t11\\t16\\t1e-20\\t30\\t90.2\\n" )\n- pathF.write( "2\\tchr1\\t1001\\t1006\\tTE2\\t11\\t16\\t1e-20\\t30\\t90.2\\n" )\n- pathF.write( "2\\tchr1\\t1201\\t1226\\tTE2\\t10\\t26\\t1e-40\\t70\\t87.2\\n" )\n- pathF.close()\n- p1 = Path()\n- p1.setFromString( "2\\tchr1\\t1001\\t1006\\tTE2\\t11\\t16\\t1e-20\\t30\\t90.2\\n" )\n- p2 = Pat'..b'()\n- self.assertEqual( expList, obsList )\n- self._db.dropTable( obsTable )\n- \n- \n- def test_path2PathRangeFromQuery_QryDirSbjRev( self ):\n- self._db.createTable( self._table, "path" )\n- p1 = Path()\n- p1.setFromTuple( ( "1", "chr1", "1", "10", "TE3", "11", "17", "1e-20", "30", "85.0" ) )\n- p2a = Path()\n- p2a.setFromTuple( ( "2", "chr2", "1", "100", "TE2", "109", "10", "1e-20", "163", "92.1" ) )\n- p2b = Path()\n- p2b.setFromTuple( ( "2", "chr2", "201", "250", "TE2", "200", "151", "1e-10", "75", "88.7" ) )\n- for p in [ p1, p2a, p2b ]: self._tpA.insert( p )\n- p2 = Path()\n- p2.setFromTuple( ( "2", "chr2", "1", "250", "TE2", "200", "10", "1e-20", "238", "90.96" ) ) # \'merge\' p2a and p2b\n- expList = [ p2 ]\n- obsTable = self._tpA._path2PathRangeFromQuery( "chr2" )\n- self._tpA._table = obsTable\n- obsList = self._tpA.getListOfAllPaths()\n- self.assertEqual( obsList, expList )\n- self._db.dropTable( obsTable )\n- \n- \n- def test_getNbOccurrences( self ):\n- self._db.createTable( self._table, "path" )\n- p1 = Path()\n- p1.setFromTuple( ( "1", "chr1", "1", "10", "TE3", "11", "17", "1e-20", "30", "85.0" ) )\n- \n- exp = 0\n- obs = self._tpA.getNbOccurrences( p1 )\n- self.assertEquals( exp, obs )\n- \n- self._tpA.insert( p1 )\n- exp = 1\n- obs = self._tpA.getNbOccurrences( p1 )\n- self.assertEquals( exp, obs )\n- \n- self._tpA.insert( p1 )\n- exp = 2\n- obs = self._tpA.getNbOccurrences( p1 )\n- self.assertEquals( exp, obs )\n- \n- def test_getListOfUniqueOccPath(self):\n- \n- p1 = Path()\n- p1.setFromTuple( ( "1", "chr1", "1", "10", "TE3", "11", "17", "1e-20", "30", "85.0" ) )\n- p2 = Path()\n- p2.setFromTuple( ( "1", "chr1", "1", "10", "TE3", "11", "17", "1e-20", "30", "85.0" ) )\n- p3 = Path()\n- p3.setFromTuple( ( "1", "chr1", "2", "10", "TE3", "11", "17", "1e-20", "30", "85.0" ) )\n- p4 = Path()\n- p4.setFromTuple( ( "2", "chr2", "2", "11", "TE4", "10", "18", "1e-30", "40", "95.0" ) )\n- lPath = [p1,p2,p3,p4]\n- \n- expListPath = deepcopy([p1,p3,p4]) \n- obsListUniquePath = self._tpA.getListOfUniqueOccPath(lPath)\n- self.assertEquals( expListPath, obsListUniquePath )\n-\n- def test_getListOfUniqueOccPath_empty_list(self):\n- expListPath = [] \n- obsListUniquePath = self._tpA.getListOfUniqueOccPath([])\n- self.assertEquals( expListPath, obsListUniquePath )\n- \n- def test_getListOfUniqueOccPath_one_item(self):\n- p1 = Path()\n- p1.setFromTuple( ( "1", "chr1", "1", "10", "TE3", "11", "17", "1e-20", "30", "85.0" ) )\n- expListPath = deepcopy([p1]) \n- obsListUniquePath = self._tpA.getListOfUniqueOccPath([p1])\n- self.assertEquals( expListPath, obsListUniquePath )\n-\n- def test_getListOfUniqueOccPath_unsorted_list(self):\n- \n- p1 = Path()\n- p1.setFromTuple( ( "1", "chr1", "1", "10", "TE3", "11", "17", "1e-20", "30", "85.0" ) )\n- p3 = Path()\n- p3.setFromTuple( ( "1", "chr1", "3", "10", "TE3", "11", "17", "1e-20", "30", "85.0" ) )\n- p4 = Path()\n- p4.setFromTuple( ( "2", "chr2", "2", "11", "TE4", "10", "18", "1e-30", "40", "95.0" ) )\n- p2 = Path()\n- p2.setFromTuple( ( "1", "chr1", "1", "10", "TE3", "11", "17", "1e-20", "30", "85.0" ) )\n- \n- lPath = [p1,p3,p4,p2]\n- \n- expListPath = deepcopy([p1,p3,p4]) \n- obsListUniquePath = self._tpA.getListOfUniqueOccPath(lPath)\n- self.assertEquals( expListPath, obsListUniquePath )\n-\n-test_suite = unittest.TestSuite()\n-test_suite.addTest( unittest.makeSuite( Test_TablePathAdaptator ) )\n-if __name__ == "__main__":\n- unittest.TextTestRunner(verbosity=2).run( test_suite )\n' |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/test/Test_TableSeqAdaptator.py --- a/commons/core/sql/test/Test_TableSeqAdaptator.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| b'@@ -1,321 +0,0 @@\n-# Copyright INRA (Institut National de la Recherche Agronomique)\n-# http://www.inra.fr\n-# http://urgi.versailles.inra.fr\n-#\n-# This software is governed by the CeCILL license under French law and\n-# abiding by the rules of distribution of free software. You can use, \n-# modify and/ or redistribute the software under the terms of the CeCILL\n-# license as circulated by CEA, CNRS and INRIA at the following URL\n-# "http://www.cecill.info". \n-#\n-# As a counterpart to the access to the source code and rights to copy,\n-# modify and redistribute granted by the license, users are provided only\n-# with a limited warranty and the software\'s author, the holder of the\n-# economic rights, and the successive licensors have only limited\n-# liability. \n-#\n-# In this respect, the user\'s attention is drawn to the risks associated\n-# with loading, using, modifying and/or developing or reproducing the\n-# software by the user in light of its specific status of free software,\n-# that may mean that it is complicated to manipulate, and that also\n-# therefore means that it is reserved for developers and experienced\n-# professionals having in-depth computer knowledge. Users are therefore\n-# encouraged to load and test the software\'s suitability as regards their\n-# requirements in conditions enabling the security of their systems and/or \n-# data to be ensured and, more generally, to use and operate it in the \n-# same conditions as regards security. \n-#\n-# The fact that you are presently reading this means that you have had\n-# knowledge of the CeCILL license and that you accept its terms.\n-\n-\n-import unittest\n-import os\n-import time\n-from commons.core.sql.DbMySql import DbMySql\n-from commons.core.sql.TableSeqAdaptator import TableSeqAdaptator\n-from commons.core.seq.Bioseq import Bioseq\n-from commons.core.coord.Set import Set\n-from commons.core.utils.FileUtils import FileUtils\n-\n-\n-class Test_TableSeqAdaptator( unittest.TestCase ):\n- \n- def setUp( self ):\n- self._uniqId = "%s_%s" % ( time.strftime("%Y%m%d%H%M%S") , os.getpid() )\n- self.fileUtils = FileUtils()\n- self._configFileName = "dummyConfigFile_%s" % ( self._uniqId )\n- configF = open(self._configFileName, "w" )\n- configF.write( "[repet_env]\\n" )\n- configF.write( "repet_host: %s\\n" % ( os.environ["REPET_HOST"] ) )\n- configF.write( "repet_user: %s\\n" % ( os.environ["REPET_USER"] ) )\n- configF.write( "repet_pw: %s\\n" % ( os.environ["REPET_PW"] ) )\n- configF.write( "repet_db: %s\\n" % ( os.environ["REPET_DB"] ) )\n- configF.write( "repet_port: %s\\n" % ( os.environ["REPET_PORT"] ) )\n- configF.close()\n- self._db = DbMySql( cfgFileName=self._configFileName )\n- self._table = "dummySeqTable_%s" % ( self._uniqId )\n- self._tsA = TableSeqAdaptator( self._db, self._table )\n- \n- \n- def tearDown( self ):\n- self._db.dropTable( self._table )\n- self._db.close()\n- os.remove( self._configFileName )\n- self._configFileName = ""\n- \n- \n-##################################################################################\n-################## Tests for methods in ITableSeqAdaptator #######################\n-##################################################################################\n- \n- def test_insert( self ):\n- bs = Bioseq( "seq1", "AGCGATGACGATGCGAGT" )\n- self._db.createTable( self._table, "fasta" )\n- self._tsA.insert( bs )\n- \n- expBioseqTuple = (("seq1", "AGCGATGACGATGCGAGT", "seq1", 18L), )\n- \n- sqlCmd = "SELECT * FROM %s" % ( self._table )\n- self._db.execute( sqlCmd )\n- obsBioseqTuple = self._db.cursor.fetchall()\n- \n- self.assertEqual( expBioseqTuple, obsBioseqTuple )\n- \n- \n- def test_insertList( self ):\n- bs1 = Bioseq( "seq1 desc", "AGCGATGACGATGCGAGT" )\n- bs2 = Bioseq( "seq2", "AGCGATGACGATGCGAGT")\n- '..b'")\n- inF.write(">seq2\\n")\n- inF.write("GCGATGCAGATGACGGCGGATGC\\n")\n- inF.close()\n- self._db.createTable( self._table, "fasta", inFileName )\n- lSeq1 = ("seq1", 18)\n- lSeq2 = ("seq2", 23)\n- lExp = [lSeq1,lSeq2]\n- lObs = self._tsA.getAccessionAndLengthList()\n- self.assertEqual( lObs, lExp )\n- os.remove( inFileName )\n- \n- \n- def test_getSeqLengthFromAccessionWithSingleQuote( self ):\n- inFileName = "dummyFaFile_%s" % ( self._uniqId )\n- inF = open( inFileName, "w" )\n- inF.write(">seq1\'\\n")\n- inF.write("AGCGATGACGATGCGAGT\\n")\n- inF.write(">seq2\\n")\n- inF.write("GCGATGCAGATGACGGCGGATGC\\n")\n- inF.close()\n- self._db.createTable( self._table, "fasta", inFileName )\n- exp = 18\n- obs = self._tsA.getSeqLengthFromAccession( "seq1\'" )\n- self.assertEqual( obs, exp )\n- os.remove( inFileName )\n- \n- \n- def test_getSubSequence_directStrand( self ):\n- self._db.createTable( self._table, "seq" )\n- chr = Bioseq()\n- chr.setHeader( "chr2" )\n- chr.setSequence( "AAAAAAAAAATTTTTGGGGGGGGGG" )\n- self._tsA.insert( chr )\n- exp = "TTTGGG"\n- obs = self._tsA.getSubSequence( "chr2", 13, 18 )\n- self.assertEqual( exp, obs )\n- \n- \n- def test_getSubSequence_reverseStrand( self ):\n- self._db.createTable( self._table, "seq" )\n- chr = Bioseq()\n- chr.setHeader( "chr2" )\n- chr.setSequence( "AAAAAAAAAATTTTTGGGGGGGGGG" )\n- self._tsA.insert( chr )\n- exp = "CCCAAA"\n- obs = self._tsA.getSubSequence( "chr2", 18, 13 )\n- self.assertEqual( exp, obs )\n- \n- \n- def test_getBioseqFromSetList_directStrand( self ):\n- self._db.createTable( self._table, "seq" )\n- chr = Bioseq()\n- chr.setHeader( "chr2" )\n- chr.setSequence( "AAAAAAAAAATTTTTGGGGGGGGGG" )\n- self._tsA.insert( chr )\n- lSets = []\n- lSets.append( Set( 3, "Dm-B-G600-Map3_classI-LTR-incomp", "chr2", 1, 10 ) )\n- lSets.append( Set( 3, "Dm-B-G600-Map3_classI-LTR-incomp", "chr2", 16, 25 ) )\n- exp = Bioseq( "Dm-B-G600-Map3_classI-LTR-incomp::3 chr2 1..10,16..25", "AAAAAAAAAAGGGGGGGGGG" )\n- obs = self._tsA.getBioseqFromSetList( lSets )\n- self.assertEqual( exp, obs )\n- \n- \n- def test_getBioseqFromSetList_reverseStrand( self ):\n- self._db.createTable( self._table, "seq" )\n- chr = Bioseq()\n- chr.setHeader( "chr2" )\n- chr.setSequence( "AAAAAAAAAATTTTTGGGGGGGGGG" )\n- self._tsA.insert( chr )\n- lSets = []\n- lSets.append( Set( 3, "Dm-B-G600-Map3_classI-LTR-incomp", "chr2", 10, 1 ) )\n- lSets.append( Set( 3, "Dm-B-G600-Map3_classI-LTR-incomp", "chr2", 25, 16 ) )\n- exp = Bioseq( "Dm-B-G600-Map3_classI-LTR-incomp::3 chr2 25..16,10..1", "CCCCCCCCCCTTTTTTTTTT" )\n- obs = self._tsA.getBioseqFromSetList( lSets )\n- self.assertEqual( exp, obs )\n- \n- \n- def test_isAccessionInTable_true( self ):\n- self._db.createTable( self._table, "seq" )\n- chr = Bioseq()\n- chr.setHeader( "chr2" )\n- chr.setSequence( "AAAAAAAAAATTTTTGGGGGGGGGG" )\n- self._tsA.insert( chr )\n- \n- obs = self._tsA.isAccessionInTable( "chr2" )\n- self.assertTrue( obs )\n- \n- \n- def test_isAccessionInTable_false( self ):\n- self._db.createTable( self._table, "seq" )\n- chr = Bioseq()\n- chr.setHeader( "chr2" )\n- chr.setSequence( "AAAAAAAAAATTTTTGGGGGGGGGG" )\n- self._tsA.insert( chr )\n- \n- obs = self._tsA.isAccessionInTable( "chr1" )\n- self.assertFalse( obs )\n- \n- \n-test_suite = unittest.TestSuite()\n-test_suite.addTest( unittest.makeSuite( Test_TableSeqAdaptator ) )\n-if __name__ == "__main__":\n- unittest.TextTestRunner(verbosity=2).run( test_suite )\n' |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/test/Test_TableSetAdaptator.py --- a/commons/core/sql/test/Test_TableSetAdaptator.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| b'@@ -1,330 +0,0 @@\n-# Copyright INRA (Institut National de la Recherche Agronomique)\n-# http://www.inra.fr\n-# http://urgi.versailles.inra.fr\n-#\n-# This software is governed by the CeCILL license under French law and\n-# abiding by the rules of distribution of free software. You can use, \n-# modify and/ or redistribute the software under the terms of the CeCILL\n-# license as circulated by CEA, CNRS and INRIA at the following URL\n-# "http://www.cecill.info". \n-#\n-# As a counterpart to the access to the source code and rights to copy,\n-# modify and redistribute granted by the license, users are provided only\n-# with a limited warranty and the software\'s author, the holder of the\n-# economic rights, and the successive licensors have only limited\n-# liability. \n-#\n-# In this respect, the user\'s attention is drawn to the risks associated\n-# with loading, using, modifying and/or developing or reproducing the\n-# software by the user in light of its specific status of free software,\n-# that may mean that it is complicated to manipulate, and that also\n-# therefore means that it is reserved for developers and experienced\n-# professionals having in-depth computer knowledge. Users are therefore\n-# encouraged to load and test the software\'s suitability as regards their\n-# requirements in conditions enabling the security of their systems and/or \n-# data to be ensured and, more generally, to use and operate it in the \n-# same conditions as regards security. \n-#\n-# The fact that you are presently reading this means that you have had\n-# knowledge of the CeCILL license and that you accept its terms.\n-\n-\n-import unittest\n-import time\n-import os\n-from commons.core.sql.TableSetAdaptator import TableSetAdaptator\n-from commons.core.sql.DbMySql import DbMySql\n-from commons.core.coord.Set import Set\n-\n-\n-class Test_TableSetAdaptator( unittest.TestCase ):\n-\n- def setUp( self ):\n- self._uniqId = "%s_%s" % ( time.strftime("%Y%m%d%H%M%S") , os.getpid() )\n- self._configFileName = "dummyConfigFile_%s" % ( self._uniqId )\n- configF = open(self._configFileName, "w" )\n- configF.write( "[repet_env]\\n" )\n- configF.write( "repet_host: %s\\n" % ( os.environ["REPET_HOST"] ) )\n- configF.write( "repet_user: %s\\n" % ( os.environ["REPET_USER"] ) )\n- configF.write( "repet_pw: %s\\n" % ( os.environ["REPET_PW"] ) )\n- configF.write( "repet_db: %s\\n" % ( os.environ["REPET_DB"] ) )\n- configF.write( "repet_port: %s\\n" % ( os.environ["REPET_PORT"] ) )\n- configF.close()\n- self._iDb = DbMySql( cfgFileName=self._configFileName )\n- self._table = "dummySetTable_%s" % ( self._uniqId )\n- self._tSetA = TableSetAdaptator( self._iDb, self._table )\n- \n- def tearDown( self ):\n- self._uniqId = None\n- self._iDb.dropTable( self._table )\n- self._iDb.close()\n- self._table = None\n- self._tSetA = None\n- os.remove( self._configFileName )\n- self._configFileName = ""\n-\n- def test_insert(self):\n- set2Insert = Set()\n- set2Insert.id = 1\n- set2Insert.name = "name1"\n- set2Insert.seqname = "name2"\n- set2Insert.start = 1L\n- set2Insert.end = 50L\n- self._iDb.createTable( self._table, "set", "" )\n- self._tSetA.insert( set2Insert, False )\n- sqlCmd = "SELECT * FROM %s" % ( self._table )\n- self._iDb.execute( sqlCmd )\n- expTsetTuple = ((1, "name1", "name2", 1L, 50L),)\n- obsTsetTuples = self._iDb.cursor.fetchall()\n- self.assertEquals(expTsetTuple, obsTsetTuples )\n- \n- def test_insertList ( self ):\n- self._iDb.createTable( self._table, "set", "" )\n- set1 = Set()\n- set1.setFromString( "1\\tname1\\tdesc1\\t1\\t120\\n" )\n- set2 = Set()\n- set2.setFromString( "2\\tname2\\tdesc2\\t1\\t20\\n" )\n- lset = [ set1, set2 ]\n- self._tSetA.insertList( lset )\n- sqlCmd = "SELECT * FROM %s" % ( self._table )\n- '..b'3 ]: self._tSetA.insert( m )\n- lId2del = []\n- self._tSetA.deleteFromIdList(lId2del)\n- expTSetTuples = ((1L, \'name1\', \'desc1\', 1L, 120L), (2L, \'name2\', \'desc2\', 1L, 20L), (3L, \'name2\', \'desc3\', 1L, 50L))\n- sqlCmd = "SELECT * FROM %s" % ( self._table )\n- self._iDb.execute( sqlCmd )\n- obsTsetTuples = self._iDb.cursor.fetchall()\n- \n- self.assertEqual( expTSetTuples, obsTsetTuples )\n- \n- def test_joinTwoSets(self):\n- self._iDb.createTable( self._table, "set", "" )\n- idSet1 = 5\n- set1 = Set()\n- set1.setFromString( "5\\tname1\\tdesc1\\t1\\t120\\n" ) \n- idSet2 = 2\n- set2 = Set()\n- set2.setFromString( "2\\tname2\\tdesc2\\t1\\t20\\n" )\n- lset = [ set1, set2 ]\n- self._tSetA.insertList( lset )\n- self._tSetA.joinTwoSets(idSet1, idSet2)\n- sqlCmd = "SELECT * FROM %s" % ( self._table )\n- self._iDb.execute( sqlCmd )\n- \n- expTSetTuples = ((2L, "name1", "desc1", 1L, 120L ), (2L, "name2", "desc2", 1L, 20L ))\n- obsTSetTuples = self._iDb.cursor.fetchall()\n- \n- self.assertEqual( expTSetTuples, obsTSetTuples)\n- self._iDb.dropTable(self._table)\n- \n- def test_joinTwoSetsWhereId1InfId2(self):\n- self._iDb.createTable( self._table, "set", "" )\n- idSet1 = 2\n- set1 = Set()\n- set1.setFromString( "5\\tname1\\tdesc1\\t1\\t120\\n" ) \n- \n- idSet2 = 5\n- set2 = Set()\n- set2.setFromString( "2\\tname2\\tdesc2\\t1\\t20\\n" )\n- \n- lset = [ set1, set2 ]\n- self._tSetA.insertList( lset )\n-\n- self._tSetA.joinTwoSets(idSet1, idSet2)\n- \n- sqlCmd = "SELECT * FROM %s" % ( self._table )\n- self._iDb.execute( sqlCmd )\n- \n- expTSetTuples = ((2L, "name1", "desc1", 1L, 120L ), (2L, "name2", "desc2", 1L, 20L ))\n- obsTSetTuples = self._iDb.cursor.fetchall()\n- \n- self.assertEqual( expTSetTuples, obsTSetTuples)\n- self._iDb.dropTable(self._table)\n- \n- def test_getNewId(self):\n- self._iDb.createTable( self._table, "set", "" )\n- set1 = Set()\n- set1.setFromString( "1\\tname1\\tdesc1\\t1\\t120\\n" ) \n- set2 = Set()\n- set2.setFromString( "2\\tname2\\tdesc2\\t1\\t20\\n" )\n- set3 = Set()\n- set3.setFromString( "5\\tname1\\tdesc1\\t1\\t120\\n" ) \n- set4 = Set()\n- set4.setFromString( "8\\tname2\\tdesc2\\t1\\t20\\n" )\n- lset = [ set1, set2, set3, set4 ]\n- self._tSetA.insertList( lset )\n- expId = 9\n- obsId = self._tSetA.getNewId()\n- self.assertEqual( expId, obsId)\n- self._iDb.dropTable(self._table)\n- \n- def test_getNewId_set_null(self):\n- self._iDb.createTable( self._table, "set", "" )\n- set1 = Set()\n- lset = [ set1 ]\n- self._tSetA.insertList( lset )\n- expId = 1\n- obsId = self._tSetA.getNewId()\n- self.assertEqual( expId, obsId)\n- self._iDb.dropTable(self._table) \n- \n- def test_getListOfAllSets( self ):\n- self._iDb.createTable( self._table, "set" )\n- s1 = Set()\n- s1.setFromString( "1\\tchr1\\tTE3\\t1\\t10\\n" )\n- s2a = Set()\n- s2a.setFromString( "2\\tchr1\\tTE2\\t2\\t9\\n" )\n- s2b = Set()\n- s2b.setFromString( "2\\tchr1\\tTE2\\t12\\t19\\n" )\n- lSets = [ s1, s2a, s2b ]\n- self._tSetA.insertList( lSets )\n- expLSets = [ s1, s2a, s2b ]\n- obsLSets = self._tSetA.getListOfAllSets()\n- self.assertEqual( expLSets, obsLSets )\n- \n- def test_getListOfAllSets_empty_table( self ):\n- self._iDb.createTable( self._table, "set" )\n- expList = []\n- obsList = self._tSetA.getListOfAllSets()\n- self.assertEqual( expList, obsList ) \n- \n-test_suite = unittest.TestSuite()\n-test_suite.addTest( unittest.makeSuite( Test_TableSetAdaptator ) ) \n-if __name__ == "__main__":\n- unittest.TextTestRunner(verbosity=2).run( test_suite )\n' |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/test/Tst_F_RepetJob.py --- a/commons/core/sql/test/Tst_F_RepetJob.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| b'@@ -1,236 +0,0 @@\n-import os\n-import time\n-import sys\n-import stat\n-import unittest\n-import glob\n-from commons.core.sql.DbMySql import DbMySql\n-from commons.core.sql.RepetJob import RepetJob\n-from commons.core.sql.Job import Job\n-\n-class Test_F_RepetJob(unittest.TestCase):\n-\n- def setUp(self):\n- self._jobTableName = "dummyJobTable"\n- self._db = DbMySql()\n- self._iRepetJob = RepetJob()\n- self._configFileName = "dummyConfigFile"\n- configF = open(self._configFileName, "w" )\n- configF.write( "[repet_env]\\n" )\n- configF.write( "repet_host: %s\\n" % ( os.environ["REPET_HOST"] ) )\n- configF.write( "repet_user: %s\\n" % ( os.environ["REPET_USER"] ) )\n- configF.write( "repet_pw: %s\\n" % ( os.environ["REPET_PW"] ) )\n- configF.write( "repet_db: %s\\n" % ( os.environ["REPET_DB"] ) )\n- configF.write( "repet_port: %s\\n" % ( os.environ["REPET_PORT"] ) )\n- configF.close()\n-\n- def tearDown(self):\n- self._iRepetJob = None\n- self._db.dropTable( self._jobTableName )\n- self._db.close()\n- os.remove(self._configFileName)\n- \n- def test_submitJob_with_multiple_jobs(self):\n- job1 = self._createJobInstance("job1")\n- self._createLauncherFile(job1)\n-\n- job2 = self._createJobInstance("job2")\n- self._createLauncherFile(job2)\n-\n- job3 = self._createJobInstance("job3")\n- self._createLauncherFile(job3)\n- \n- self._iRepetJob.submitJob( job1, maxNbWaitingJobs=3, checkInterval=5, verbose=0 )\n- self._iRepetJob.submitJob( job2, maxNbWaitingJobs=3, checkInterval=5, verbose=0 )\n- self._iRepetJob.submitJob( job3, maxNbWaitingJobs=3, checkInterval=5, verbose=0 )\n-\n- time.sleep(70)\n- \n- expJobStatus = "finished"\n- obsJobStatus1 = self._iRepetJob.getJobStatus(job1)\n- obsJobStatus2 = self._iRepetJob.getJobStatus(job2)\n- obsJobStatus3 = self._iRepetJob.getJobStatus(job3)\n- \n- self.assertEquals(expJobStatus, obsJobStatus1)\n- self.assertEquals(expJobStatus, obsJobStatus2)\n- self.assertEquals(expJobStatus, obsJobStatus3)\n- \n- jobName1 = job1.jobname\n- jobName2 = job2.jobname\n- jobName3 = job3.jobname\n- \n- expErrorFilePrefix1 = jobName1+ ".e" \n- expOutputFilePrefix1 = jobName1 + ".o"\n- expErrorFilePrefix2 = jobName2 + ".e" \n- expOutputFilePrefix2 = jobName2 + ".o"\n- expErrorFilePrefix3 = jobName3 + ".e" \n- expOutputFilePrefix3 = jobName3 + ".o"\n- \n- lErrorFiles1 = glob.glob(expErrorFilePrefix1 + "*")\n- lOutputFiles1 = glob.glob(expOutputFilePrefix1 + "*")\n- lErrorFiles2 = glob.glob(expErrorFilePrefix2 + "*")\n- lOutputFiles2 = glob.glob(expOutputFilePrefix2 + "*")\n- lErrorFiles3 = glob.glob(expErrorFilePrefix3 + "*")\n- lOutputFiles3 = glob.glob(expOutputFilePrefix3 + "*")\n- \n- isLErrorFileNotEmpty1 = (len(lErrorFiles1) != 0) \n- isLOutputFileNotEmpty1 = (len(lOutputFiles1) != 0)\n- isLErrorFileNotEmpty2 = (len(lErrorFiles2) != 0) \n- isLOutputFileNotEmpty2 = (len(lOutputFiles2) != 0)\n- isLErrorFileNotEmpty3 = (len(lErrorFiles3) != 0) \n- isLOutputFileNotEmpty3 = (len(lOutputFiles3) != 0)\n- \n- os.system("rm launcherFileTest*.py *.e* *.o*")\n- self.assertTrue(isLErrorFileNotEmpty1 and isLOutputFileNotEmpty1)\n- self.assertTrue(isLErrorFileNotEmpty2 and isLOutputFileNotEmpty2)\n- self.assertTrue(isLErrorFileNotEmpty3 and isLOutputFileNotEmpty3)\n-\n- def test_submitJob_job_already_submitted(self):\n- self._iRepetJob.createTable(self._jobTableName, "jobs")\n- iJob = self._createJobInstance("job")\n- self._iRepetJob.recordJob(iJob)\n- \n- isSysExitRaised = False\n- try:\n- self._iRepetJob.submitJob(iJob)\n- except SystemExit:\n- isSysExitRaised = True\n- self.'..b'ordJob(iJob)\n- self._iRepetJob.changeJobStatus(iJob, "error", "method")\n- \n- self._iRepetJob.waitJobGroup(self._jobTableName ,iJob.groupid, 0, 2)\n- \n- time.sleep(10)\n- \n- expJobStatus = "finished"\n- obsJobStatus1 = self._iRepetJob.getJobStatus(iJob)\n- \n- self.assertEquals(expJobStatus, obsJobStatus1)\n- \n- jobName = iJob.jobname\n- \n- expErrorFilePrefix1 = jobName + ".e" \n- expOutputFilePrefix1 = jobName + ".o"\n- \n- lErrorFiles1 = glob.glob(expErrorFilePrefix1 + "*")\n- lOutputFiles1 = glob.glob(expOutputFilePrefix1 + "*")\n- \n- isLErrorFileNotEmpty1 = (len(lErrorFiles1) != 0) \n- isLOutputFileNotEmpty1 = (len(lOutputFiles1) != 0)\n- \n- self._iRepetJob.removeJob(iJob) \n- os.system("rm launcherFileTest*.py *.e* *.o*")\n- self.assertTrue(isLErrorFileNotEmpty1 and isLOutputFileNotEmpty1)\n- \n-\n- def test_isJobStillHandledBySge_True(self):\n- self._iRepetJob.createTable(self._jobTableName, "jobs")\n- iJob = self._createJobInstance("job")\n- self._createLauncherFile(iJob)\n- self._iRepetJob.submitJob(iJob)\n- \n- isJobHandledBySge = self._iRepetJob.isJobStillHandledBySge(iJob.jobid, iJob.jobname)\n- os.system("rm launcherFileTest*.py")\n- \n- self.assertTrue(isJobHandledBySge)\n-\n- def test_isJobStillHandledBySge_False(self):\n- self._iRepetJob.createTable(self._jobTableName, "jobs")\n- iJob = self._createJobInstance("job")\n- self._createLauncherFile(iJob)\n- self._iRepetJob.recordJob(iJob)\n- \n- isJobHandledBySge = self._iRepetJob.isJobStillHandledBySge(iJob.jobid, iJob.jobname)\n- os.system("rm launcherFileTest*.py")\n- \n- self.assertFalse(isJobHandledBySge)\n- \n- def _createJobInstance(self, name):\n- return Job(self._jobTableName, 0, name, "test", "", "date;sleep 5;date", "./launcherFileTest_"+ name +".py")\n- \n- def _createLauncherFile(self, iJob):\n- jobFileHandler = open( iJob.launcher , "w" )\n-\n- launcher = "#!/usr/bin/python\\n"\n- launcher += "import os\\n"\n- launcher += "import sys\\n"\n- \n- launcher += "print \\"system:\\", os.uname()\\n"\n- launcher += "sys.stdout.flush()\\n"\n- newStatus = "running"\n- prg = "%s/bin/srptChangeJobStatus.py" % (os.environ["REPET_PATH"])\n- cmd = prg\n- cmd += " -t %s" % ( iJob.tablename )\n- cmd += " -n %s" % ( iJob.jobname )\n- cmd += " -g %s" % ( iJob.groupid )\n- if iJob.queue != "":\n- cmd += " -q %s" % ( iJob.queue )\n- cmd += " -s %s" % ( newStatus )\n- cmd += " -c %s" %( self._configFileName )\n- cmd += " -v 1"\n- launcher +="os.system( \\"" + cmd + "\\" )\\n"\n- \n- launcher += "print \\"LAUNCH: "+ iJob.command + "\\"\\n"\n- launcher += "sys.stdout.flush()\\n"\n- launcher += "exitStatus = os.system (\\"" + iJob.command + "\\")\\n"\n- launcher += "if exitStatus != 0:\\n"\n- launcher += "\\tprint \\"ERROR: "+ iJob.command + " returned exit status \'%i\'\\" % ( exitStatus )\\n"\n- \n- newStatus = "finished"\n- prg = os.environ["REPET_PATH"] + "/bin/srptChangeJobStatus.py"\n- cmd = prg\n- cmd += " -t %s" % ( iJob.tablename )\n- cmd += " -n %s" % ( iJob.jobname )\n- cmd += " -g %s" % ( iJob.groupid )\n- if iJob.queue != "":\n- cmd += " -q %s" % ( iJob.queue )\n- cmd += " -s %s" % ( newStatus )\n- cmd += " -c %s" %( self._configFileName )\n- cmd += " -v 1"\n- launcher +="os.system( \\"" + cmd + "\\" )\\n"\n- launcher += "sys.exit(0)\\n"\n- jobFileHandler.write(launcher)\n- jobFileHandler.close()\n- os.chmod( iJob.launcher, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC )\n-\n-if __name__ == "__main__":\n- unittest.main()\n\\ No newline at end of file\n' |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/sql/test/Tst_RepetJob.py --- a/commons/core/sql/test/Tst_RepetJob.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| b'@@ -1,395 +0,0 @@\n-import unittest\n-import sys\n-import os\n-import time\n-from commons.core.sql.DbMySql import DbMySql\n-from commons.core.sql.Job import Job\n-from commons.core.sql.RepetJob import RepetJob\n-from commons.core.utils.FileUtils import FileUtils\n-\n-#TODO: to remove... => replace all RepetJob() by TableJobAdaptator()...\n-class Test_RepetJob( unittest.TestCase ):\n- \n- def setUp(self):\n- self._jobTableName = "dummyJobTable"\n- self._db = DbMySql()\n- self._iRepetJob = RepetJob()\n- \n- def tearDown(self):\n- self._iRepetJob = None\n- self._db.close()\n- \n- def _createJobInstance(self):\n- return Job( self._jobTableName, 0, "job1", "groupid", "queue", "command", "launcherFile", "node", "lResources" )\n- \n- def test_createJobTable_is_table_created(self):\n- self._iRepetJob.createTable(self._jobTableName, "jobs")\n- \n- isTableCreated = self._db.doesTableExist(self._jobTableName)\n- self.assertTrue(isTableCreated)\n- \n- self._db.dropTable(self._jobTableName)\n- \n- def test_createJobTable_field_list(self):\n- self._iRepetJob.createTable(self._jobTableName, "jobs")\n-\n- obsLFiled = self._db.getFieldList(self._jobTableName)\n- expLField = ["jobid", "jobname", "groupid", "command", "launcher", "queue", "status", "time", "node"]\n- \n- self.assertEquals(expLField, obsLFiled)\n- \n- self._db.dropTable(self._jobTableName)\n- \n- def test_recordJob(self):\n- self._iRepetJob.createTable(self._jobTableName, "jobs")\n- iJob = self._createJobInstance()\n- self._iRepetJob.recordJob(iJob)\n- \n- qryParams = "SELECT jobid, groupid, command, launcher, queue, status, node FROM " + self._jobTableName + " WHERE jobid = %s" \n- params = (iJob.jobid)\n- \n- self._db.execute(qryParams, params)\n- \n- tObs = self._db.fetchall()[0]\n- tExp =(iJob.jobid, iJob.groupid, iJob.command, iJob.launcher, iJob.queue, "waiting", "?")\n- \n- self.assertEquals(tExp,tObs)\n-\n- self._db.dropTable(self._jobTableName)\n- \n- def test_removeJob(self):\n- self._iRepetJob.createTable(self._jobTableName, "jobs")\n- iJob = self._createJobInstance()\n- self._iRepetJob.recordJob(iJob)\n-\n- self._iRepetJob.removeJob(iJob)\n- \n- isTableEmpty = self._db.isEmpty(self._jobTableName)\n- \n- self.assertTrue(isTableEmpty)\n- \n- self._db.dropTable(self._jobTableName)\n- \n- def test_getJobStatus(self):\n- self._iRepetJob.createTable(self._jobTableName, "jobs")\n- iJob = self._createJobInstance()\n- self._iRepetJob.recordJob(iJob)\n-\n- expStatus = "waiting"\n- obsStatus = self._iRepetJob.getJobStatus(iJob)\n- \n- self.assertEquals(expStatus, obsStatus)\n- self._db.dropTable(self._jobTableName)\n- \n- def test_getJobStatus_unknown(self):\n- self._iRepetJob.createTable(self._jobTableName, "jobs")\n- iJob = self._createJobInstance() \n-\n- expStatus = "unknown"\n- obsStatus = self._iRepetJob.getJobStatus(iJob)\n- \n- self.assertEquals(expStatus, obsStatus)\n- self._db.dropTable(self._jobTableName)\n- \n- def test_getJobStatus_no_name(self):\n- self._iRepetJob.createTable(self._jobTableName, "jobs")\n- iJob = Job( self._jobTableName, 20, "", "groupid", "queue", "command", "launcherFile", "node", "lResources" ) \n- \n- expStatus = "unknown"\n- obsStatus = self._iRepetJob.getJobStatus(iJob)\n- \n- self.assertEquals(expStatus, obsStatus)\n- self._db.dropTable(self._jobTableName)\n- \n- def test_getJobStatus_non_unique_job(self):\n- # Warning : this case will not append, because recordJob() begin by removeJob()\n- self._iRepetJob.createTable(self._jobTableName, "jobs")\n- iJob = self._createJobInstance()\n- sqlCmd = "INSERT I'..b'RepetJob.removeJob(iJob)\n- \n- def test_setJobIdFromSge(self):\n- self._iRepetJob.createTable(self._jobTableName, "jobs")\n- iJob = self._createJobInstance()\n- self._iRepetJob.recordJob(iJob)\n- self._iRepetJob.setJobIdFromSge(iJob, 1000)\n- \n- qryParams = "SELECT jobid FROM " + self._jobTableName + " WHERE jobname = %s AND queue = %s AND groupid = %s" \n- params = (iJob.jobname, iJob.queue, iJob.groupid)\n- \n- self._db.execute(qryParams, params)\n- \n- tObs = self._db.fetchall()[0]\n- tExp =(1000,)\n- \n- self.assertEquals(tExp,tObs)\n- \n- self._db.dropTable(self._jobTableName)\n- \n- def test_submitJob_8_fields_for_job_table(self):\n- iJob = self._createJobInstance()\n- self._db.dropTable(self._jobTableName)\n- sqlCmd = "CREATE TABLE " + self._jobTableName \n- sqlCmd += " ( jobid INT UNSIGNED"\n- sqlCmd += ", groupid VARCHAR(255)"\n- sqlCmd += ", command TEXT"\n- sqlCmd += ", launcher VARCHAR(1024)"\n- sqlCmd += ", queue VARCHAR(255)"\n- sqlCmd += ", status VARCHAR(255)"\n- sqlCmd += ", time DATETIME"\n- sqlCmd += ", node VARCHAR(255) )"\n- self._db.execute(sqlCmd)\n- \n- self._iRepetJob.submitJob(iJob)\n- \n- expFieldsNb = 9\n- obsFieldsNb = len(self._iRepetJob.getFieldList(self._jobTableName))\n- \n- self.assertEquals(expFieldsNb, obsFieldsNb)\n- \n- self._db.dropTable(self._jobTableName)\n- \n- def test_getNodesListByGroupId(self):\n- self._iRepetJob.createTable(self._jobTableName, "jobs")\n- iJob1 = Job( self._jobTableName, 0, "job1", "groupid", "queue", "command", "launcherFile", "node1", "lResources" )\n- iJob2 = Job( self._jobTableName, 1, "job2", "groupid", "queue", "command", "launcherFile", "node2", "lResources" )\n- iJob3 = Job( self._jobTableName, 2, "job3", "groupid2", "queue", "command", "launcherFile", "node3", "lResources" )\n- \n- self._insertJob(iJob1)\n- self._insertJob(iJob2)\n- self._insertJob(iJob3)\n- \n- expNodeList = ["node1", "node2"]\n- obsNodeList = self._iRepetJob.getNodesListByGroupId(self._jobTableName, "groupid")\n- self.assertEquals(expNodeList, obsNodeList)\n- \n- self._db.dropTable(self._jobTableName)\n- \n- def test_getNodesListByGroupId_empty_list(self):\n- self._iRepetJob.createTable(self._jobTableName, "jobs")\n- iJob1 = Job( self._jobTableName, 0, "job1", "groupid", "queue", "command", "launcherFile", "node1", "lResources" )\n- iJob2 = Job( self._jobTableName, 1, "job2", "groupid", "queue", "command", "launcherFile", "node2", "lResources" )\n- iJob3 = Job( self._jobTableName, 2, "job3", "groupid32", "queue", "command", "launcherFile", "node3", "lResources" )\n- \n- self._insertJob(iJob1)\n- self._insertJob(iJob2)\n- self._insertJob(iJob3)\n- \n- expNodeList = []\n- obsNodeList = self._iRepetJob.getNodesListByGroupId(self._jobTableName, "groupid3")\n- self.assertEquals(expNodeList, obsNodeList)\n- \n- self._db.dropTable(self._jobTableName)\n- \n- def _insertJob(self, iJob):\n- self._iRepetJob.removeJob( iJob )\n- sqlCmd = "INSERT INTO %s" % ( iJob.tablename )\n- sqlCmd += " VALUES ("\n- sqlCmd += " \\"%s\\"," % ( iJob.jobid )\n- sqlCmd += " \\"%s\\"," % ( iJob.jobname )\n- sqlCmd += " \\"%s\\"," % ( iJob.groupid )\n- sqlCmd += " \\"%s\\"," % ( iJob.command.replace("\\"","\\\'") )\n- sqlCmd += " \\"%s\\"," % ( iJob.launcher )\n- sqlCmd += " \\"%s\\"," % ( iJob.queue )\n- sqlCmd += " \\"waiting\\","\n- sqlCmd += " \\"%s\\"," % ( time.strftime( "%Y-%m-%d %H:%M:%S" ) )\n- sqlCmd += " \\"%s\\" );" % ( iJob.node )\n- self._iRepetJob.execute( sqlCmd )\n- \n-if __name__ == "__main__":\n- unittest.main()\n' |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/stat/Stat.py --- a/commons/core/stat/Stat.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| @@ -1,209 +0,0 @@ -import math - -class Stat(object): - - def __init__(self, lValues = []): - self.reset() - if lValues != []: - self.fill(lValues) - - def __eq__(self, o): - self._lValues.sort() - o._lValues.sort() - return self._lValues == o._lValues and round(self._sum, 6) == round(o._sum, 6) \ - and round(self._sumOfSquares, 6) == round(o._sumOfSquares, 6) and self._n == self._n \ - and round(self._min, 6) == round(o._min, 6) and round(self._max, 6) == round(o._max, 6) - - def getValuesList(self): - return self._lValues - - def getSum(self): - return self._sum - - def getSumOfSquares(self): - return self._sumOfSquares - - def getValuesNumber(self): - return self._n - - def getMin(self): - return self._min - - def getMax(self): - return self._max - - ## Reset all attributes - # - def reset(self): - self._lValues = [] - self._sum = 0.0 - self._sumOfSquares = 0.0 - self._n = 0 - self._max = 0.0 - self._min = 0.0 - - ## Add a value to Stat instance list and update attributes - # - # @param v float value to add - # - def add(self, v): - self._lValues.append( float(v) ) - self._sum += float(v) - self._sumOfSquares += float(v) * float(v) - self._n = self._n + 1 - if v > self._max: - self._max = float(v) - if self._n == 1: - self._min = float(v) - elif v < self._min: - self._min = float(v) - - ## Add a list of values to Stat instance list and update attributes - # - # @param lValues list of float list to add - # - def fill(self, lValues): - for v in lValues: - self.add(v) - - ## Get the arithmetic mean of the Stat instance list - # - # @return float - # - def mean(self): - if self._n == 0: - return 0 - else: - return self._sum / float(self._n) - - ## Get the variance of the sample - # @note we consider a sample, not a population. So for calculation, we use n-1 - # - # @return float - # - def var(self): - if self._n < 2 or self.mean() == 0.0: - return 0 - else: - variance = self._sumOfSquares/float(self._n - 1) - self._n/float(self._n - 1) * self.mean()*self.mean() - if round(variance, 10) == 0: - variance = 0 - return variance - - ## Get the standard deviation of the sample - # - # @return float - # - def sd(self): - return math.sqrt( self.var() ) - - ## Get the coefficient of variation of the sample - # - # @return float - # - def cv(self): - if self._n < 2 or self.mean() == 0.0: - return 0 - else: - return self.sd() / self.mean() - - ## Get the median of the sample - # - # @return number or "NA" (Not available) - # - def median( self ): - if len(self._lValues) == 0: - return "NA" - if len(self._lValues) == 1: - return self._lValues[0] - self._lValues.sort() - m = int( math.ceil( len(self._lValues) / 2.0 ) ) - if len(self._lValues) % 2: - return self._lValues[m-1] - else: - return ( self._lValues[m-1] + self._lValues[m] ) / 2.0 - - ## Get the kurtosis (measure of whether the data are peaked or flat relative to a normal distribution, 'coef d'aplatissement ' in french)). - # k = 0 -> completely flat - # k = 3 -> same as normal distribution - # k >> 3 -> peak - # - # @return float - # - def kurtosis(self): - numerator = 0 - for i in self._lValues: - numerator += math.pow( i - self.mean(), 4 ) - return numerator / float(self._n - 1) * self.sd() - - ## Prepare a string with calculations on your values - # - # @return string - # - def string(self): - msg = "" - msg += "n=%d" % ( self._n ) - msg += " mean=%5.3f" % ( self.mean() ) - msg += " var=%5.3f" % ( self.var() ) - msg += " sd=%5.3f" % ( self.sd() ) - msg += " min=%5.3f" % ( self.getMin() ) - median = self.median() - if median == "NA": - msg += " med=%s" % (median) - else: - msg += " med=%5.3f" % (median) - msg += " max=%5.3f" % ( self.getMax() ) - return msg - - ## Print descriptive statistics - # - def view(self): - print self.string() - - ## Return sorted list of values, ascending (default) or descending - # - # @return list - # - def sort( self, isReverse = False ): - self._lValues.sort(reverse = isReverse) - return self._lValues - - ## Give the quantile corresponding to the chosen percentage - # - # @return number - # - def quantile( self, percentage ): - if self._n == 0: - return 0 - elif percentage == 1: - return self.getMax() - else: - return self.sort()[int(self._n * percentage)] - - ## Prepare a string with quantile values - # - # @return string - # - def stringQuantiles( self ): - return "n=%d min=%5.3f Q1=%5.3f median=%5.3f Q3=%5.3f max=%5.3f" % \ - (self._n, self.quantile(0), self.quantile(0.25), self.quantile(0.5), self.quantile(0.75), self.quantile(1)) - - ## Print quantiles string - # - def viewQuantiles( self ): - print self.stringQuantiles() - - ## Compute N50 - # @return number - def N50(self ): - lSorted = self.sort(True) - midlValues = self.getSum() / 2 - cumul = 0 - index = 0 - while cumul < midlValues: - cumul = cumul + lSorted[index] - index += 1 - if (index == 0): - return lSorted[index] - else : - return lSorted[index - 1] \ No newline at end of file |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/stat/test/Test_F_Stat.py --- a/commons/core/stat/test/Test_F_Stat.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| @@ -1,22 +0,0 @@ -import unittest -from commons.core.stat.Stat import Stat - - -class Test_F_Stat(unittest.TestCase): - - - def test_output(self): - lValues = [0, -1, -5, 112, 10.2, 0.5, 4, -0.5] - iStat = Stat(lValues) - expString = "n=8 mean=15.025 var=1554.934 sd=39.433 min=-5.000 med=0.250 max=112.000" - self.assertEquals(expString, iStat.string()) - - def test_outputQuantile(self): - lValues = [0, -1, -5, 112, 10.2, 0.5, 4, -0.5] - iStat = Stat(lValues) - expString = "n=8 min=-5.000 Q1=-0.500 median=0.500 Q3=10.200 max=112.000" - self.assertEquals(expString, iStat.stringQuantiles()) - - -if __name__ == "__main__": - unittest.main() \ No newline at end of file |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/stat/test/Test_Stat.py --- a/commons/core/stat/test/Test_Stat.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| b'@@ -1,356 +0,0 @@\n-from commons.core.stat.Stat import Stat\n-import unittest\n-\n-class Test_Stat(unittest.TestCase):\n- \n- def test__eq__true(self):\n- iStat1 = Stat([1, 2, 3, 46])\n- iStat2 = Stat([1, 2, 3, 46])\n- self.assertTrue(iStat1 == iStat2)\n-\n- def test__eq__false(self):\n- iStat1 = Stat([1, 2, 3, 4])\n- iStat2 = Stat([1, 2, 3, 46])\n- self.assertFalse(iStat1 == iStat2)\n-\n- def test__eq__disordered_list(self):\n- iStat1 = Stat([3, 2, 1, 46])\n- iStat2 = Stat([1, 2, 3, 46])\n- self.assertTrue(iStat1 == iStat2)\n-\n- def test_reset(self):\n- lValues = [1, 2, 5, 9, 12, 46]\n- iStat = Stat(lValues)\n- iStat.reset()\n- expValuesList = []\n- expSum = 0\n- expSum2 = 0\n- expN = 0\n- expMin = 0\n- expMax = 0\n- obsValuesList = iStat.getValuesList()\n- obsSum = iStat.getSum()\n- obsSum2 = iStat.getSumOfSquares()\n- obsN = iStat.getValuesNumber()\n- obsMin = iStat.getMin()\n- obsMax = iStat.getMax()\n- self.assertEquals(expValuesList, obsValuesList)\n- self.assertEquals(expSum, obsSum)\n- self.assertEquals(expSum2, obsSum2)\n- self.assertEquals(expN, obsN)\n- self.assertEquals(expMin, obsMin)\n- self.assertEquals(expMax, obsMax)\n-\n- def test_add_EmptyList(self):\n- lValues = []\n- iStat = Stat(lValues)\n- iStat.add(5)\n- expValuesList = [5]\n- expSum = 5\n- expSum2 = 25\n- expN = 1\n- expMin = 5\n- expMax = 5\n- obsValuesList = iStat.getValuesList()\n- obsSum = iStat.getSum()\n- obsSum2 = iStat.getSumOfSquares()\n- obsN = iStat.getValuesNumber()\n- obsMin = iStat.getMin()\n- obsMax = iStat.getMax()\n- self.assertEquals(expValuesList, obsValuesList)\n- self.assertEquals(expSum, obsSum)\n- self.assertEquals(expSum2, obsSum2)\n- self.assertEquals(expN, obsN)\n- self.assertEquals(expMin, obsMin)\n- self.assertEquals(expMax, obsMax)\n- \n- def test_add_Max(self):\n- lValues = [0,1,1]\n- iStat = Stat(lValues)\n- iStat.add(2)\n- expValuesList = [0,1,1,2]\n- expSum = 4\n- expSum2 = 6\n- expN = 4\n- expMin = 0\n- expMax = 2\n- obsValuesList = iStat.getValuesList()\n- obsSum = iStat.getSum()\n- obsSum2 = iStat.getSumOfSquares()\n- obsN = iStat.getValuesNumber()\n- obsMin = iStat.getMin()\n- obsMax = iStat.getMax()\n- self.assertEquals(expValuesList, obsValuesList)\n- self.assertEquals(expSum, obsSum)\n- self.assertEquals(expSum2, obsSum2)\n- self.assertEquals(expN, obsN)\n- self.assertEquals(expMin, obsMin)\n- self.assertEquals(expMax, obsMax)\n- \n- def test_add_Min(self):\n- lValues = [2,1,1]\n- iStat = Stat(lValues)\n- iStat.add(0)\n- expValuesList = [2,1,1,0]\n- expSum = 4\n- expSum2 = 6\n- expN = 4\n- expMin = 0\n- expMax = 2\n- obsValuesList = iStat.getValuesList()\n- obsSum = iStat.getSum()\n- obsSum2 = iStat.getSumOfSquares()\n- obsN = iStat.getValuesNumber()\n- obsMin = iStat.getMin()\n- obsMax = iStat.getMax()\n- self.assertEquals(expValuesList, obsValuesList)\n- self.assertEquals(expSum, obsSum)\n- self.assertEquals(expSum2, obsSum2)\n- self.assertEquals(expN, obsN)\n- self.assertEquals(expMin, obsMin)\n- self.assertEquals(expMax, obsMax)\n- \n- def test_fill_emptyList(self):\n- lValues = [2,1,1]\n- iStat = Stat(lValues)\n- iStat.fill([])\n- expValuesList = [2,1,1]\n- expSum = 4\n- expSum2 = 6\n- expN = 3\n- expMin = 1\n- expMax = 2\n- obsValuesList = iStat.getValuesList()\n- obsSum = iStat.getSum()\n- obsSum2 = iStat.getSumOfSquares()\n- obsN = iStat.getValuesNumber()'..b'\n- lValues = [1, 2, 3, 4, 1, 2, 54, 6, 7]\n- iStat = Stat(lValues)\n- expMedian = 3\n- obsMedian = iStat.median()\n- self.assertEquals(expMedian, obsMedian)\n- \n- def test_median_odd(self):\n- lValues = [1, 2, 3, 4, 2, 54, 6, 7]\n- iStat = Stat(lValues)\n- expMedian = 3.5\n- obsMedian = iStat.median()\n- self.assertEquals(expMedian, obsMedian)\n- \n- def test_kurtosis_flat(self):\n- lValues = [1, 1, 1]\n- iStat = Stat(lValues)\n- expKurtosis = 0\n- obsKurtosis = iStat.kurtosis()\n- self.assertEquals(expKurtosis, obsKurtosis)\n- \n- def test_kurtosis_peak(self):\n- lValues = [1, 100, -5]\n- iStat = Stat(lValues)\n- expKurtosis = round(712872278.6609683, 2)\n- obsKurtosis = round(iStat.kurtosis(), 2)\n- self.assertEquals(expKurtosis, obsKurtosis)\n- \n- def test_kurtosis_normal(self):\n- lValues = [-1, 0, 1.64, 1.64, 0, -1]\n- iStat = Stat(lValues)\n- expKurtosis = 3.0\n- obsKurtosis = round(iStat.kurtosis(), 1)\n- self.assertEquals(expKurtosis, obsKurtosis)\n- \n- def test_sort(self):\n- lValues = [-1, 0, 1.64, 1.64, 0, -1]\n- iStat = Stat(lValues)\n- expSort = [-1, -1, 0, 0, 1.64, 1.64]\n- obsSort = iStat.sort()\n- self.assertEquals(expSort, obsSort)\n- \n- def test_sort_reverse(self):\n- lValues = [-1, 0, 1.64, 1.64, 0, -1]\n- iStat = Stat(lValues)\n- expSort = [1.64, 1.64, 0, 0, -1, -1]\n- obsSort = iStat.sort(True)\n- self.assertEquals(expSort, obsSort)\n- \n- def test_sort_emptyList(self):\n- lValues = []\n- iStat = Stat(lValues)\n- expSort = []\n- obsSort = iStat.sort()\n- self.assertEquals(expSort, obsSort)\n- \n- def test_quantile_emptyList(self):\n- lValues = []\n- iStat = Stat(lValues)\n- expQuantile = 0\n- obsQuantile = iStat.quantile(0.25)\n- self.assertEquals(expQuantile, obsQuantile)\n- \n- def test_quantile_0perc(self):\n- lValues = [0, 2.64, 1.64, -1, 5]\n- iStat = Stat(lValues)\n- expQuantile = -1\n- obsQuantile = iStat.quantile(0)\n- self.assertEquals(expQuantile, obsQuantile)\n- \n- def test_quantile_25perc(self):\n- lValues = [0, 2.64, 1.64, -1, 5]\n- iStat = Stat(lValues)\n- expQuantile = 0\n- obsQuantile = iStat.quantile(0.25)\n- self.assertEquals(expQuantile, obsQuantile)\n- \n- def test_quantile_41perc(self):\n- lValues = [0, 2.64, 1.64, -1, 5]\n- iStat = Stat(lValues)\n- expQuantile = 1.64\n- obsQuantile = iStat.quantile(0.41)\n- self.assertEquals(expQuantile, obsQuantile)\n- \n- def test_quantile_75perc(self):\n- lValues = [0, 2.64, 1.64, -1, 5]\n- iStat = Stat(lValues)\n- expQuantile = 2.64\n- obsQuantile = iStat.quantile(0.75)\n- self.assertEquals(expQuantile, obsQuantile)\n- \n- def test_quantile_81perc(self):\n- lValues = [0, 2.64, 1.64, -1, 5]\n- iStat = Stat(lValues)\n- expQuantile = 5\n- obsQuantile = iStat.quantile(0.81)\n- self.assertEquals(expQuantile, obsQuantile)\n- \n- def test_quantile_100perc(self):\n- lValues = [0, 2.64, 1.64, -1, 5]\n- iStat = Stat(lValues)\n- expQuantile = 5\n- obsQuantile = iStat.quantile(1)\n- self.assertEquals(expQuantile, obsQuantile)\n- \n- def test_N50(self):\n- lValues = [10, 10, 2, 16, 3, 4, 5]\n- iStat = Stat(lValues)\n- expN50 = 10\n- obsN50 = iStat.N50()\n- self.assertEquals(expN50, obsN50)\n-\n- def test_N50SpecialValues(self):\n- lValues = [1, 100, 2, 3]\n- iStat = Stat(lValues)\n- expN50 = 100\n- obsN50 = iStat.N50()\n- self.assertEquals(expN50, obsN50)\n- \n-if __name__ == "__main__":\n- unittest.main()\n\\ No newline at end of file\n' |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/test/Test_LoggerFactory.py --- a/commons/core/test/Test_LoggerFactory.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| b |
| @@ -1,93 +0,0 @@ -import unittest -import logging -from commons.core.LoggerFactory import LoggerFactory - -class Test_LoggerFactory( unittest.TestCase ): - - def test_logger_debug(self): - iLogger = LoggerFactory.createLogger("test") - isMethodExecuted = True - try: - iLogger.debug("message") - except: - isMethodExecuted = False - self.assertTrue(isMethodExecuted) - - def test_logger_info(self): - iLogger = LoggerFactory.createLogger("test") - isMethodExecuted = True - try: - iLogger.info("message") - except: - isMethodExecuted = False - self.assertTrue(isMethodExecuted) - - def test_logger_warning(self): - iLogger = LoggerFactory.createLogger("test") - isMethodExecuted = True - try: - iLogger.warning("message") - except: - isMethodExecuted = False - self.assertTrue(isMethodExecuted) - - def test_logger_error(self): - iLogger = LoggerFactory.createLogger("test") - isMethodExecuted = True - try: - iLogger.error("message") - except: - isMethodExecuted = False - self.assertTrue(isMethodExecuted) - - def test_logger_level_debug(self): - iLogger = LoggerFactory.createLogger("test") - LoggerFactory.setLevel(iLogger, 4) - expLevel = logging.DEBUG - obsLevel = iLogger.getEffectiveLevel() - self.assertEquals(expLevel, obsLevel) - - def test_logger_level_info(self): - iLogger = LoggerFactory.createLogger("test") - LoggerFactory.setLevel(iLogger, 3) - expLevel = logging.INFO - obsLevel = iLogger.getEffectiveLevel() - self.assertEquals(expLevel, obsLevel) - - def test_logger_level_warning(self): - iLogger = LoggerFactory.createLogger("test") - LoggerFactory.setLevel(iLogger, 2) - expLevel = logging.WARNING - obsLevel = iLogger.getEffectiveLevel() - self.assertEquals(expLevel, obsLevel) - - def test_logger_level_error(self): - iLogger = LoggerFactory.createLogger("test") - LoggerFactory.setLevel(iLogger, 1) - expLevel = logging.ERROR - obsLevel = iLogger.getEffectiveLevel() - self.assertEquals(expLevel, obsLevel) - - def test_logger_default_level(self): - iLogger = LoggerFactory.createLogger("test") - expLevel = logging.ERROR - obsLevel = iLogger.getEffectiveLevel() - self.assertEquals(expLevel, obsLevel) - - def test_logger_quiet(self): - iLogger = LoggerFactory.createLogger("test") - LoggerFactory.setLevel(iLogger, 0) - self.assertTrue(iLogger.disabled) - - def test_logger_noduplicate_handler(self): - iLogger = LoggerFactory.createLogger("test") - iLogger2 = LoggerFactory.createLogger("test") - - expNbHandlers = 1 - obsNbHandlers = len(iLogger2.handlers) - self.assertEquals(expNbHandlers, obsNbHandlers) - -test_suite = unittest.TestSuite() -test_suite.addTest( unittest.makeSuite( Test_LoggerFactory ) ) -if __name__ == "__main__": - unittest.TextTestRunner(verbosity=2).run( test_suite ) \ No newline at end of file |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/tree/Tree.py --- a/commons/core/tree/Tree.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| @@ -1,122 +0,0 @@ -import os, re, sys - -class Tree: - - def __init__( self, inFileName="" ): - self.tree = None - self.inFileName = inFileName - if self.inFileName != "": - self.loadTree() - - def loadTree( self, verbose=0 ): - inF = open( self.inFileName, "r" ) - lines = inF.readlines() - inF.close() - line = "".join(lines).replace("\n","") - self.tree = self.parseTree( line ) - if verbose > 0: - print "nb of leaves: %i" % ( self.getNbOfLeaves( self.tree ) ) - - def parseTree( self, sTree ): - if "," not in sTree: - name, length = sTree.split(":") - return self.makeLeaf( name, float(length) ) - - distPattern = re.compile(r'(?P<tree>\(.+\))\:(?P<length>[e\-\d\.]+)$') - m = distPattern.search( sTree ) - length = 0 - if m: - if m.group('length'): length = float( m.group('length') ) - sTree = m.group('tree') - if length == "": length = 0 - - lhs, rhs = self.parseSubTree( sTree ) - - return { "name": "internal", - "left": self.parseTree( lhs ), - "right": self.parseTree( rhs ), - "length": length } - - def makeLeaf( self, name, length ): - return { "left":None, "right":None, "name":name, "length":length } - - def parseSubTree( self, sTree ): - """ - Parse a newick-formatted string of type 'a,b' into [a,b] - """ - chars = list( sTree[1:-1] ) - count = 0 - isLhs = True - leftS = "" - rightS = "" - for c in chars: - if c == "(": - count += 1 - elif c == ")": - count -= 1 - elif (c == ",") and (count == 0) and (isLhs) : - isLhs = False - continue - if isLhs: leftS += c - else: rightS += c - return [ leftS, rightS ] - - def toNewick( self, tree ): - newString = "" - if tree["name"] is not "internal": - newString += tree["name"] - else: - newString += "(" - newString += self.toNewick( tree["left"] ) - newString += "," - newString += self.toNewick( tree["right"] ) - newString += ")" - if tree["length"]: - newString += ":" - newString += "%f" % ( tree["length"] ) - return newString - - def saveTree( self, outFileName ): - outF = open( outFileName, "w" ) - outF.write( self.toNewick( self.tree ) ) - outF.close() - - def replaceHeaderViaPrefixSearch( self, tree, dNew2Init ): - if dNew2Init.has_key( tree["name"] ): - tree["name"] = dNew2Init[ tree["name"] ].replace(" ","_").replace("::","-").replace(",","-") - if tree["left"] != None: - self.replaceHeaderViaPrefixSearch( tree["left"], dNew2Init ) - if tree["right"] != None: - self.replaceHeaderViaPrefixSearch( tree["right"], dNew2Init ) - - def retrieveInitialSequenceHeaders( self, dNew2Init, outFileName ): - tree = self.tree - self.replaceHeaderViaPrefixSearch( tree, dNew2Init ) - self.tree = tree - self.saveTree( outFileName ) - - def getNbOfChildNodes( self, tree, nbNodes ): - if tree["left"] is not None: - nbNodes += 1 - nbNodes = self.getNbOfChildNodes( tree["left"], nbNodes ) - if tree["right"] is not None: - nbNodes += 1 - nbNodes = self.getNbOfChildNodes( tree["right"], nbNodes ) - return nbNodes - - def getNbOfNodes( self ): - nbNodes = 0 - return self.getNbOfChildNodes( self.tree, nbNodes ) - - def getNbOfChildLeaves( self, tree, nbLeaves ): - if tree["name"] != "internal": - nbLeaves += 1 - if tree["left"] is not None: - nbLeaves = self.getNbOfChildLeaves( tree["left"], nbLeaves ) - if tree["right"] is not None: - nbLeaves = self.getNbOfChildLeaves( tree["right"], nbLeaves ) - return nbLeaves - - def getNbOfLeaves( self ): - nbLeaves = 0 - return self.getNbOfChildLeaves( self.tree, nbLeaves ) |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/tree/test/Test_Tree.py --- a/commons/core/tree/test/Test_Tree.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| @@ -1,90 +0,0 @@ -import unittest -import os -import time -from commons.core.tree.Tree import Tree -from commons.core.utils.FileUtils import FileUtils - - -class Test_Tree( unittest.TestCase ): - - def setUp( self ): - self._tree = Tree() - self._uniqId = "%s_%s" % ( time.strftime("%Y%m%d%H%M%S") , os.getpid() ) - - - def test_parseTree_oneLeaf( self ): - inString = "seq1:0.0023" - obs = self._tree.parseTree( inString ) - exp = { "left":None, "right":None, "name":"seq1", "length":0.0023 } - self.assertEqual( obs, exp ) - - - def test_parseTree_twoLeaves( self ): - inString = "(seq1:0.0023,seq2:0.0017)" - obs = self._tree.parseTree( inString ) - exp = {'length':0, 'right':{'length':0.0016999999999999999, 'right':None, 'name':'seq2', 'left':None}, 'name':'internal', 'left':{'length':0.0023, 'right':None, 'name':'seq1', 'left':None}} - self.assertEqual( obs, exp ) - -## def test_parseTree_threeLeaves( self ): -## inString = "(seq1:0.0023,(seq2:0.0017,seq3:0.0009))" -## obs = self._tree.parseTree( inString ) -## print obs -## exp = {'length':0, 'right':{'length':0.0016999999999999999, 'right':None, 'name':'seq2', 'left':None}, 'name':'internal', 'left':{'length':0.0023, 'right':None, 'name':'seq1', 'left':None}} -## self.assertEqual( obs, exp ) - - - def test_parseSubTree( self ): - inString = "(seq1:0.0023,seq2:0.0017)" - lExp = [ "seq1:0.0023", "seq2:0.0017" ] - lObs = self._tree.parseSubTree( inString ) - self.assertEqual( lObs, lExp ) - - - def test_saveTree( self ): - inFileName = "dummyInFile_%s" % ( self._uniqId ) - inF = open( inFileName, "w" ) - inF.write( "(seq4:0.012511,(seq3:0.005340,seq2:0.002201))" ) - inF.close() - self._tree = Tree( inFileName ) - obsFileName = "dummyObsFile_%s" % ( self._uniqId ) - self._tree.saveTree( obsFileName ) - self.assertTrue( FileUtils.are2FilesIdentical( obsFileName, inFileName ) ) - for f in [ inFileName, obsFileName ]: - os.remove( f ) - - - def test_retrieveInitialSequenceHeaders( self ): - inString = "(seq4:0.012511,(seq3:0.005340,seq2:0.002201))" - self._tree.tree = self._tree.parseTree( inString ) - dNew2Init = { "seq2":"consensus524::215 dmel_chr4 142..765", "seq3":"DmelChr4-B-G387-MAP16", "seq4":"1360|1cl-3gr" } - expFileName = "dummyExpFile_%s" % ( self._uniqId ) - expF = open( expFileName, "w" ) - expF.write( "(1360|1cl-3gr:0.012511,(DmelChr4-B-G387-MAP16:0.005340,consensus524-215_dmel_chr4_142..765:0.002201))" ) - expF.close() - obsFileName = "dummyObsFile_%s" % ( self._uniqId ) - self._tree.retrieveInitialSequenceHeaders( dNew2Init, obsFileName ) - self.assertTrue( FileUtils.are2FilesIdentical( obsFileName, expFileName ) ) - for f in [ expFileName, obsFileName ]: - os.remove( f ) - - - def test_getNbOfLeaves( self ): - inString = "(seq4:0.012511,(seq3:0.005340,seq2:0.002201))" - self._tree.tree = self._tree.parseTree( inString ) - exp = 3 - obs = self._tree.getNbOfLeaves() - self.assertEqual( obs, exp ) - - - def test_getNbOfNodes( self ): - inString = "(seq4:0.012511,(seq3:0.005340,seq2:0.002201))" - self._tree.tree = self._tree.parseTree( inString ) - exp = 4 - obs = self._tree.getNbOfNodes() - self.assertEqual( obs, exp ) - - -test_suite = unittest.TestSuite() -test_suite.addTest( unittest.makeSuite( Test_Tree ) ) -if __name__ == "__main__": - unittest.TextTestRunner(verbosity=2).run( test_suite ) |
| b |
| diff -r 3441fe98a2ba -r aa0420172fc6 commons/core/tree/test/treeTestSuite.py --- a/commons/core/tree/test/treeTestSuite.py Tue Apr 30 14:34:10 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| b |
| @@ -1,16 +0,0 @@ -import unittest -import sys -import Test_Tree - - - -def main(): - - commonsTestSuite = unittest.TestSuite() - commonsTestSuite.addTest(unittest.makeSuite(Test_Tree.Test_Tree,'test')) - runner = unittest.TextTestRunner(sys.stderr, 2, 2) - runner.run(commonsTestSuite) - - -if __name__ == '__main__': - main() |