Galaxy |

Changeset 0:b126ea31824f (2017-02-06)

Commit message:
1st Uploaded

added:
README.txt
TEannot.sh
TEannot.xml
TEannot_lite.py
tool_dependencies.xml

diff -r 000000000000 -r b126ea31824f README.txt
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/README.txt Mon Feb 06 13:37:49 2017 -0500

@@ -0,0 +1,63 @@
+REPET dependency
+
+For REPET version 2.5.1 (REPET with some patchs, please contact urgi-contact@versailles.inra.fr to get it)
+
+REPET dependency handle the creation of essential environment variables.
+
+!!!!WARNING : REPET has to be installed!!!!
+
+
+After you have install your tool with this dependency, please go to check if defined environment variables suit with your configuration.
+Check this file :
+***tool_dependency_dir***/repet/2.5/vmarcon/package_repet_2_5/***revision***/env.sh
+Please replace with correct values :
+ * tool_dependency_dir (value in galaxy.ini file)
+ * revision
+
+Then in this file (env.sh) modify variable value in order to adjust it to your
+system :
+ - Database connexion (REPET_HOST, REPET_USER, REPET_PW, REPET_DB, REPET_PORT)
+ - Job manager (REPET_JOB_MANAGER, REPET_QUEUE)
+ - Working environment (REPET_PATH, REPET_NUCL_BANK, REPET_PROT_BANK, REPET_HMM_PROFILES, REPET_RDNA_BANK)
+
+
+If you want REPET working in a specific temporary directory, fill the variable REPET_TMP_DIR.
+
+If you don't want to use one or several databanks, remove the corresponding variable.
+
+
+
+-------------
+ Galaxy Page
+-------------
+
+!!!! To get the content of the page and example datasets, please ask it sendig an e-mail at urgi-contact@versailles.inra.fr . !!!!
+
+
+
+To explain in detail to your users how TEannot_lite works, please make a galaxy Page:
+- Connect in your Galaxy.
+- Go to the "Saved Pages" (User > Saved Pages).
+- Create a new page ("Add new Page" button in the top right corner) named 'teannot'.
+- Click on "teannot" and "Edit content".
+- Paste the content of the URGI TEannot page.
+- Save.
+
+Now you have to create the two Embed Galaxy Object (example on the blue and green area).
+- On the URGI TEannot page, download "DmelChr4.fa", "TElib_DmelChr4.fa" and "TElib_DmelCrh4.classif" by clicking on "Save dataset" (green area).
+- Upload this file on your Galaxy instance where REPET Lite is installed IN A NEW HISTORY.
+- Rename these datasets as in example "DmelChr4.fa", "TElib_DmelChr4.fa" and "TElib_DmelCrh4.classif".
+- Launch "Repet Lite - TEannot" with "DmelChr4.fa" as Fasta alignment input, "TElib_DmelChr4.fa" as Fasta TE library and "TElib_DmelCrh4.classif" as Classification file.
+- Go back to the page in edition mode.
+- Add a green area - Embed Datasets - DmelChr4.fa
+- Add a green area - Embed Datasets - TElib_DmelChr4.fa
+- Add a green area - Embed Datasets - TElib_DmelChr4.classif
+- Add the blue area - Embed Histories - your history with TEannot result.
+- Save again.
+
+To publish your page:
+- Go to the list of "Saved Pages"
+- Click on "teannot" and "Share or Publish"
+- And "Make Page Accessible and Publish"
+
+Now this page is accessible with the URL shown and in Shared Data > Published Pages

diff -r 000000000000 -r b126ea31824f TEannot.sh
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/TEannot.sh Mon Feb 06 13:37:49 2017 -0500

[

@@ -0,0 +1,50 @@
+#!/bin/bash
+set -e
+
+fasta=$1
+library=$2
+outputfile=$3
+outputmaskedfile=$4
+outputlog=$5
+outputconfig=$6
+outputStats=$7
+classif=$8
+outputmasked_SSRmaskfile=$9
+
+projectname=$(date "+%Y%m%d")
+
+add=''
+
+if [ -f $classif ]
+then
+  add='-c '$classif
+fi
+
+if [ -f $outputStats ]
+then
+  add=$add' -s'
+fi
+
+`dirname $0`'/'TEannot_lite.py -i $fasta -l $library -o $outputfile $add > $outputlog
+projectname_complete=$(ls $(pwd)|grep $projectname)
+working_dir=$(pwd)/$projectname_complete
+sed -i 's@'"$working_dir"'@'$projectname'@g' $outputlog
+mv $outputfile-$projectname.gff3 $outputfile
+mv $outputfile-$projectname.mask $outputmaskedfile
+mv $outputfile-$projectname.mask_SSRmask.fa $outputmasked_SSRmaskfile
+if [ -f $outputStats ]
+then
+  mv $outputfile-$projectname-TEstats.txt $outputStats
+fi
+
+workingconfigfile=$working_dir/TEannot_Galaxy_config_$projectname_complete
+sed -i 's|repet_host:.*|repet_host:|g' $workingconfigfile
+sed -i 's|repet_user:.*|repet_user:|g' $workingconfigfile
+sed -i 's|repet_pw:.*|repet_pw:|g' $workingconfigfile
+sed -i 's|repet_db:.*|repet_db:|g' $workingconfigfile
+sed -i 's|repet_port:.*|repet_port:|g' $workingconfigfile
+sed -i 's|repet_job_manager:.*|repet_job_manager:|g' $workingconfigfile
+sed -i 's|project_name:.*|project_name: '$projectname'|g' $workingconfigfile
+sed -i 's|project_dir:.*|project_dir:|g' $workingconfigfile
+sed -i 's|tmpDir:.*|tmpDir:|g' $workingconfigfile
+mv $workingconfigfile $outputconfig

diff -r 000000000000 -r b126ea31824f TEannot.xml
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/TEannot.xml Mon Feb 06 13:37:49 2017 -0500

[

b'@@ -0,0 +1,327 @@\n+<tool id="teannot" name="REPET Lite - TEannot" version="1.5.0">\n+ \n+ \n+ <description> Genome annotation for masking transposable elements</description>\n+ \n+ \n+ <requirements>\n+ <requirement type="binary">python</requirement>\n+ <requirement type="package" version="2.5">repet</requirement>\n+ </requirements>\n+ \t \n+ \n+ <stdio>\n+ \n+ <exit_code range="1:" level="fatal"/>\n+ <exit_code range=":-1" level="fatal"/>\n+\n+ </stdio>\n+\n+\n+ \n+ <version_command>\n+ \n+ </version_command>\n+ \n+ \n+ <command interpreter="bash">\n+ \tTEannot.sh $fasta $library $outputfile $outputmaskedfile $outputlog $outputconfig\n+ \t#if str( $withStats ) == "yes":\n+ \t$outputstatsfile\n+ \t#else : \n+ \t$withStats\n+ \t#end if\n+ \t$classif\n+ $outputmasked_SSRmaskfile\n+ </command>\n+ \t \n+ \n+ <inputs>\n+ \t<param name="fasta" type="data" format="fasta" optional="false" label="Fasta alignment input" />\n+ \t<param name="library" type="data" format="fasta" optional="false" label="Fasta TE library [from TEdenovo]" />\n+ \t<param name="classif" type="data" format="tabular" optional="true" label="Classification file" help="To add classification information in the output file." />\n+ \t<param name="label" type="text" label="Output name" />\n+ \t<param name="withStats" type="select" label="Get statistical file">\n+ \t <option value="yes" selected="true">Yes</option>\n+ \t <option value="no">No</option>\n+ \t</param>\n+ </inputs> \n+ \t\t\t\t \t \t \t \t \t \t \t \t \n+ \n+ <outputs>\n+\t<data name="outputlog" type="data" format="txt" label="TEannot-#if str($label)==\'\' then $fasta.name else $label #.log" />\n+ \t<data name="outputfile" type="data" format="gff3" label="TEannot-#if str($label)==\'\' then $fasta.name else $label #.gff3" />\n+ \t<data name="outputmaskedfile" type="data" format="fasta" label="TEannot-#if str($label)==\'\' then $fasta.name else $label #_masked.fa" />\n+ \t<data name="outputmasked_SSRmaskfile" type="data" format="fasta" label="TEannot-#if str($label)==\'\' then $fasta.name else $label #_SSRmask.fa" />\n+ \t<data name="outputstatsfile" type="data" format="txt" label="TEannot-#if str($label)==\'\' then $fasta.name else $label #.stats" >\n+ \t\t<filter>(withStats == \'yes\')</filter>\n+ \t</data>\n+ \t<data name="outputconfig" type="data" format="txt" label="TEannot-#if str($label)==\'\' then $fasta.name else $label #.cfg" />\n+ </outputs>\n+ \n+\n+ \n+ <tests>\n+ \n+ <test>\n+ <param name="fasta" value="alignment.fa" />\n+ <param name="library" value="libTE.fa" />\n+ <output name="outputfile" >\n+ \t<assert_contents>\n+\t\t\t<has_line_matching expression="^##gff-version 3" />\n+\t\t\t<has_n_columns n="9" />\n+ \t</assert_contents>\n+ </output>\n+ <output name="outputmaskedfile" >\n+ \t<assert_contents>\n+ \t\t<has_line_matching expression="^>\\w+" />\n+ \t\t<has_line_matching expression="[ACTGX]{60}" />\n+ \t</assert_contents>\n+ </output>\n+ <output name="outputstatsfile">\n+ \t<assert_contents>\n+ \t\t<has_line_matching expre'..b'in <italic>De Novo</italic> Annotation Approaches},\n+ year = {2011},\n+ month = {01},\n+ volume = {6},\n+ url = {http://dx.doi.org/10.1371%2Fjournal.pone.0016526},\n+ pages = {e16526},\n+ abstract = {\n+ <p>Transposable elements (TEs) are mobile, repetitive DNA sequences that are almost ubiquitous in prokaryotic and eukaryotic genomes. They have a large impact on genome structure, function and evolution. With the recent development of high-throughput sequencing methods, many genome sequences have become available, making possible comparative studies of TE dynamics at an unprecedented scale. Several methods have been proposed for the <italic>de novo</italic> identification of TEs in sequenced genomes. Most begin with the detection of genomic repeats, but the subsequent steps for defining TE families differ. High-quality TE annotations are available for the <italic>Drosophila melanogaster</italic> and <italic>Arabidopsis thaliana</italic> genome sequences, providing a solid basis for the benchmarking of such methods. We compared the performance of specific algorithms for the clustering of interspersed repeats and found that only a particular combination of algorithms detected TE families with good recovery of the reference sequences. We then applied a new procedure for reconciling the different clustering results and classifying TE sequences. The whole approach was implemented in a pipeline using the REPET package. Finally, we show that our combined approach highlights the dynamics of well defined TE families by making it possible to identify structural variations among their copies. This approach makes it possible to annotate TE families and to study their diversification in a single analysis, improving our understanding of TE dynamics at the whole-genome scale and for diverse species.</p>\n+ },\n+ number = {1},\n+ doi = {10.1371/journal.pone.0016526}\n+ }]]></citation>\n+ <citation type="bibtex"><![CDATA[@article{10.1371/journal.pone.0094101,\n+ author = {Maumus, Florian AND Quesneville, Hadi},\n+ journal = {PLoS ONE},\n+ publisher = {Public Library of Science},\n+ title = {Deep Investigation of <italic>Arabidopsis thaliana</italic> Junk DNA Reveals a Continuum between Repetitive Elements and Genomic Dark Matter},\n+ year = {2014},\n+ month = {04},\n+ volume = {9},\n+ url = {http://dx.doi.org/10.1371%2Fjournal.pone.0094101},\n+ pages = {e94101},\n+ abstract = {<p>Eukaryotic genomes contain highly variable amounts of DNA with no apparent function. This so-called junk DNA is composed of two components: repeated and repeat-derived sequences (together referred to as the repeatome), and non-annotated sequences also known as genomic dark matter. Because of their high duplication rates as compared to other genomic features, transposable elements are predominant contributors to the repeatome and the products of their decay is thought to be a major source of genomic dark matter. Determining the origin and composition of junk DNA is thus important to help understanding genome evolution as well as host biology. In this study, we have used a combination of tools enabling to show that the repeatome from the small and reducing <italic>A. thaliana</italic> genome is significantly larger than previously thought. Furthermore, we present the concepts and results from a series of innovative approaches suggesting that a significant amount of the <italic>A. thaliana</italic> dark matter is of repetitive origin. As a tentative standard for the community, we propose a deep compendium annotation of the <italic>A. thaliana</italic> repeatome that may help addressing farther genome evolution as well as transcriptional and epigenetic regulation in this model plant.</p>},\n+ number = {4},\n+ doi = {10.1371/journal.pone.0094101}\n+ }]]></citation>\n+ </citations>\n+ \n+</tool>\n'

diff -r 000000000000 -r b126ea31824f TEannot_lite.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/TEannot_lite.py Mon Feb 06 13:37:49 2017 -0500

[

b'@@ -0,0 +1,245 @@\n+#!/usr/bin/env python\n+\n+import os\n+import sys\n+import time\n+import glob\n+import shutil\n+import ConfigParser\n+import re\n+\n+if not "REPET_PATH" in os.environ.keys():\n+\tprint("ERROR: no environment variable REPET_PATH")\n+\tsys.exit(1)\n+\n+if (not "REPET_DB" in os.environ.keys()) or (not "REPET_HOST" in os.environ.keys()) or (not "REPET_PORT" in os.environ.keys()) or (not "REPET_USER" in os.environ.keys()) or (not "REPET_PW" in os.environ.keys()):\n+\tprint "ERROR: there is at least one environment database variable missing : REPET_DB, REPET_PORT, REPET_HOST, REPET_USER or REPET_PW"\n+\tsys.exit(1)\n+\n+if not "REPET_JOB_MANAGER" in os.environ.keys():\n+\tprint "ERROR: no environment variable REPET_JOB_MANAGER"\n+\tsys.exit(1)\n+\n+\n+if not "%s/bin" % os.environ["REPET_PATH"] in os.environ["PATH"]: \n+\tos.environ["PATH"] = "%s/bin:%s" % (os.environ["REPET_PATH"], os.environ["PATH"])\n+\n+sys.path.append(os.environ["REPET_PATH"])\n+if not "PYTHONPATH" in os.environ.keys():\n+\tos.environ["PYTHONPATH"] = os.environ["REPET_PATH"]\n+else:\n+\tos.environ["PYTHONPATH"] = "%s:%s" % (os.environ["REPET_PATH"], os.environ["PYTHONPATH"])\n+\n+\n+from commons.core.LoggerFactory import LoggerFactory\n+from commons.core.checker.RepetException import RepetException\n+from commons.core.utils.FileUtils import FileUtils\n+from commons.core.utils.RepetOptionParser import RepetOptionParser\n+from commons.core.seq.FastaUtils import * #FastaUtils\n+from commons.core.sql.DbFactory import DbFactory\n+\n+LOG_DEPTH = "TEannot.pipeline"\n+\n+class TEannot_lite(object):\n+\n+\tdef __init__(self, configFileName = "", fastaFileName = "", libraryFileName = "", verbosity = 0):\n+\t\tself._configFileName = configFileName\n+\t\tself._fastaFileName = os.path.abspath(fastaFileName)\n+\t\tself._libraryFileName = os.path.abspath(libraryFileName)\n+\t\tself._projectName = time.strftime("%Y%m%d%H%M%S")\n+\t\tself._outputGff = ""\n+\t\tself._classif = ""\n+\t\t#self._maskedThreshold = 80\n+\t\tself._statsFile = ""\n+\t\tself._outputMasked = ""\n+\t\tif "REPET_TMP_DIR" in os.environ.keys():\n+ \t self._tmp_dir = os.environ["REPET_TMP_DIR"]\n+\t\telse :\n+ \t self._tmp_dir = ""\n+\t\tself._verbosity = verbosity\n+\t\tself._log = LoggerFactory.createLogger("%s.%s" % (LOG_DEPTH, self.__class__.__name__), self._verbosity)\n+\n+\tdef setAttributesFromCommandLine(self):\n+\t\tdescription = "This script is a ligth version of TEannot. It writes configuration file and launches TEannot."\n+\t\tepilog = "Example: TEannot_lite.py -i fastaFileName -l fastaLibraryFileName \\n"\n+\t\tversion = "1.1"\n+\t\tparser = RepetOptionParser(description = description, epilog = epilog, version = version)\n+\t\tparser.add_option("-i", "--fasta", dest = "fastaFileName" , action = "store" , type = "string", help ="Input fasta file name ", default = "")\n+\t\tparser.add_option("-l", "--lib", dest = "libraryFileName" , action = "store" , type = "string", help ="Input fasta library file name ", default = "")\n+\t\tparser.add_option("-c", "--withClassif", dest = "withClassif" , action = "store" , type = "string" , metavar="CLASSIFFILE" , help ="[optional] To add classification informations in GFF3 file, please put classif file from TEdenovo step. ", default = "")\n+\t\t#parser.add_option("-t", "--maskedThreshold", dest = "maskedThreshold" , action = "store", type = "int", metavar="80", help ="[optional] [default: 80] To choose the threshold of the identity percent for the masked fasta file. ", default = 80)\n+\t\tparser.add_option("-s", "--stats", dest="withStats", action="store_true",help = " Get statistical file in output.", default = False)\n+\t\tparser.add_option("-o", "--output", dest="outputLabel" , action = "store", type = "string", help = " [optional] Label for GFF3 output file", default = "")\n+\t\tparser.add_option("-v", "--verbosity", dest = "verbosity", action = "store", type = "int", metavar="2", help = "Verbosity [optional] [default: 2]", default = 2)\n+\t\toptions = parser.parse_args()[0]\n+\t\tself._setAttributesFromOptions(options)\n+\n+\tdef _setAttrib'..b'lCmds.append( "TEannot.py -P %s -C %s -S 2 -a RM -r -v %i" % (self._projectName, self._configFileName, self._verbosity) ) #\n+\t\tlCmds.append( "TEannot.py -P %s -C %s -S 2 -a CEN -r -v %i" % (self._projectName, self._configFileName, self._verbosity) ) #\n+\t\tlCmds.append( "TEannot.py -P %s -C %s -S 4 -s TRF -v %i" % (self._projectName, self._configFileName, self._verbosity) )\n+\t\tlCmds.append( "TEannot.py -P %s -C %s -S 4 -s RMSSR -v %i" % (self._projectName, self._configFileName, self._verbosity) )\n+\t\tlCmds.append( "TEannot.py -P %s -C %s -S 4 -s Mreps -v %i" % (self._projectName, self._configFileName, self._verbosity) )\n+\t\tlCmds.append( "TEannot.py -P %s -C %s -S 5 -v %i" % (self._projectName, self._configFileName, self._verbosity) )\n+\t\tlCmds.append( "TEannot.py -P %s -C %s -S 3 -c BLR+RM+CEN -v %i" % (self._projectName, self._configFileName, self._verbosity) )\n+\t\tlCmds.append( "TEannot.py -P %s -C %s -S 7 -v %i" % (self._projectName, self._configFileName, self._verbosity) )\n+\t\tlCmds.append( "TEannot.py -P %s -C %s -S 8 -v %i -o GFF3" % (self._projectName, self._configFileName, self._verbosity) )\n+\n+\t\tif self._classif!=\'\':\n+\t\t\tself._setClassifTable()\n+\t\n+\t\tfor cmd in lCmds:\n+\t\t\treturnValue = os.system(cmd)\n+\t\tif returnValue != 0:\n+\t\t\tprint "ERROR: command \'%s\' returned %i" % (cmd, returnValue)\n+\t\t\tself._cleanTables()\n+\t\t\tsys.exit(1)\n+\t\n+\t\tprint "END time: %s" % time.strftime("%Y-%m-%d %H:%M:%S")\n+\n+\n+\tdef _maskFasta(self):\n+\t\tpathFile = self._outputMasked+"_tmp.path"\n+\t\tsetFile = self._outputMasked+"_tmp.set"\n+\t\tlCmds = []\n+\t\tlCmds.append("srptExportTable.py -i %s_chr_allTEs_nr_noSSR_path -C %s -o %s -v %s" % (self._projectName,self._configFileName,pathFile,self._verbosity))\t\n+\t\tlCmds.append("MaskSeqFromCoord.py -i %s -m %s -f path -X -o %s -v %s" % (self._fastaFileName,pathFile,self._outputMasked,self._verbosity))\n+\t\tlCmds.append("srptExportTable.py -i %s_chr_allSSRs_set -C %s -o %s -v %s " % (self._projectName,self._configFileName, setFile,self._verbosity))\n+\t\tlCmds.append("MaskSeqFromCoord.py -i %s -m %s -f set -X -o %s_SSRmask.fa -v %s" % (self._outputMasked, setFile, self._outputMasked, self._verbosity))\n+\n+\t\tfor cmd in lCmds:\n+\t\t\treturnValue = os.system(cmd)\n+\t\t\tif returnValue != 0:\n+\t\t\t\tprint "ERROR: command \'%s\' returned %i" % (cmd, returnValue)\n+\t\t\t\tself._cleanTables()\n+\t\t\t\tsys.exit(1)\n+\n+\t\t#os.system("rm -f %s"%pathFile) \n+\n+\tdef _createStatsFile(self):\n+\t\tfastaFile=open(self._fastaFileName)\n+\t\tfastaLength=FastaUtils.dbCumLength( fastaFile )\n+\t\tcmd = "PostAnalyzeTELib.py -a 3 -g {0} -p {1}_chr_allTEs_nr_noSSR_path -s {1}_refTEs_seq".format(fastaLength,self._projectName) \n+\t\tos.system(cmd)\n+\t\tcmd = "mv %s_chr_allTEs_nr_noSSR_path.globalAnnotStatsPerTE.txt %s"%(self._projectName,self._statsFile)\n+\t\tos.system(cmd)\n+\t\t\n+\tdef _setClassifTable(self):\n+\t\tiDb = DbFactory.createInstance()\n+\t\tiDb.createTable("%s_consensus_classif" % self._projectName, "classif", self._classif, True)\n+\t\tiDb.close()\n+\n+\tdef _launchListAndDropTables(self):\t\n+\t\tcmd = "ListAndDropTables.py"\n+\t\tcmd += " -C %s" % self._configFileName\n+\t\tcmd += " -d \'%s\'" % self._projectName\n+\t\tos.system(cmd)\n+\n+\tdef _cleanJobsTable(self):\n+\t\tdb = DbFactory.createInstance( configFileName = self._configFileName ) \n+\t\tsql_cmd="DELETE FROM jobs WHERE groupid like \'%s%%\';"%self._projectName\n+\t\tdb.execute( sql_cmd )\n+\t\tdb.close()\n+\n+\tdef _cleanTables(self):\n+\t\tself._launchListAndDropTables()\t\n+\t\tself. _cleanJobsTable()\n+\n+\tdef run(self):\n+\t\tos.mkdir(self._projectName)\n+\t\tos.chdir(self._projectName)\n+\t\tself._writeConfigFile()\n+\t\tos.symlink(self._fastaFileName,"%s/%s.fa" %(os.getcwd(),self._projectName)) #creer repertoire projet\n+\t\tos.symlink(self._libraryFileName,"%s/%s_refTEs.fa" %(os.getcwd(),self._projectName))\n+\t\tself._launchTEannot()\n+\t\tself._mergeOutputGff()\n+\t\tself._maskFasta()\n+\t\tif self._statsFile : \n+\t\t\tself._createStatsFile()\n+\t\tself._cleanTables()\n+\n+if __name__ == \'__main__\':\n+\tiTEannot= TEannot_lite()\n+\tiTEannot.setAttributesFromCommandLine()\n+\tiTEannot.run()\n'

diff -r 000000000000 -r b126ea31824f tool_dependencies.xml
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/tool_dependencies.xml Mon Feb 06 13:37:49 2017 -0500

@@ -0,0 +1,6 @@
+<?xml version="1.0"?>
+<tool_dependency>
+ <package name="repet" version="2.5">
+ <repository changeset_revision="f3b6009634b1" name="package_repet_2_5" owner="vmarcon" toolshed="https://toolshed.g2.bx.psu.edu" />
+ </package>
+</tool_dependency>