Galaxy |

Changeset 0:baea09e6722b (2017-02-06)

Commit message:
1st Uploaded

added:
README.txt
TEdenovo.sh
TEdenovo.xml
TEdenovo_lite.py
tool_dependencies.xml

diff -r 000000000000 -r baea09e6722b README.txt
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/README.txt Mon Feb 06 13:31:53 2017 -0500

@@ -0,0 +1,69 @@
+REPET dependency
+
+For REPET version 2.5.1 (REPET with some patchs, please contact urgi-contact@versailles.inra.fr to get it)
+
+REPET dependency handle the creation of essential environment variables.
+
+!!!!WARNING : REPET has to be installed!!!!
+
+
+After you have install your tool with this dependency, please go to check if defined environment variables suit with your configuration.
+Check this file :
+***tool_dependency_dir***/repet/2.5/vmarcon/package_repet_2_5/***revision***/env.sh
+Please replace with correct values :
+ * tool_dependency_dir (value in galaxy.ini file)
+ * revision
+
+Then in this file (env.sh) modify variable value in order to adjust it to your
+system :
+ - Database connexion (REPET_HOST, REPET_USER, REPET_PW, REPET_DB, REPET_PORT)
+ - Job manager (REPET_JOB_MANAGER, REPET_QUEUE)
+ - Working environment (REPET_PATH, REPET_NUCL_BANK, REPET_PROT_BANK, REPET_HMM_PROFILES, REPET_RDNA_BANK)
+
+If you want REPET working in a specific temporary directory, fill the variable REPET_TMP_DIR.
+
+If you don't want to use one or several databanks, remove the corresponding variable.
+
+If you notice a problem running Grouper on step 3, please complete LIB_GCC4_8_2 and LD_LIBRARY_PATH variable.
+
+
+-------------
+ Galaxy Page
+-------------
+
+!!!! To get the content of the page and example datasets, please ask it sendig an e-mail at urgi-contact@versailles.inra.fr . !!!!
+
+To explain in detail to your users how TEdenovo_lite works, please make a galaxy Page:
+- Connect in your Galaxy.
+- Go to the "Saved Pages" (User > Saved Pages).
+- Create a new page ("Add new Page" button in the top right corner) named 'tedenovo'.
+- Click on "tedenovo" and "Edit content".
+- Paste the content of the URGI TEdenovo page.
+- Save.
+
+Now you have to create the two Embed Galaxy Object (example on the blue and green area).
+- On the URGI TEdenovo page, download "DmelChr4.fa" by clicking on "Save dataset" (green area).
+- Upload this file on your Galaxy instance where REPET Lite is installed IN A NEW HISTORY.
+- Rename this dataset "DmelChr4.fa".
+- Launch "Repet Lite - TEdenovo" with "DmelChr4.fa" as Fasta alignment input, "Yes" to get classification informations.
+- Go back to the page in edition mode.
+- Add the green area - Embed Datasets - DmelChr4.fa
+- Add the blue area - Embed Histories - your history with TEdenovo result.
+- Save again.
+
+To publish your page:
+- Go to the list of "Saved Pages"
+- Click on "tedenovo" and "Share or Publish"
+- And "Make Page Accessible and Publish"
+
+Now this page is accessible with the URL shown and in Shared Data > Published Pages
+
+
+--------------------
+Vizualisation plugin
+--------------------
+
+We developped a plugin to vizualise the statistical output (".classif_stats.txt").
+Please send an e-mail at urgi-contact@versailles.inra.fr to get it.
+
+

diff -r 000000000000 -r baea09e6722b TEdenovo.sh
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/TEdenovo.sh Mon Feb 06 13:31:53 2017 -0500

[

@@ -0,0 +1,44 @@
+#!/bin/bash
+set -e
+
+fasta=$1
+outputfasta=$2
+classif=$3
+outputlog=$4
+outputclassif=$7
+outputconfig=$5
+outputstats=$6
+
+projectname=$(date "+%Y%m%d")
+
+
+add=''
+
+if [ $classif == yes ]
+then
+ add='-c'
+fi
+`dirname $0`'/'TEdenovo_lite.py -i $fasta -o $outputfasta $add > $outputlog
+projectname_complete=$(ls $(pwd)|grep $projectname)
+working_dir=$(pwd)/$projectname_complete
+sed -i 's@'"$working_dir"'@'$projectname'@g' $outputlog
+mv $outputfasta-$projectname-denovoLibTEs_filtered.fa $outputfasta
+mv $outputfasta-$projectname-classif_stats.txt $outputstats
+if [ $classif == yes ]
+then
+ mv $outputfasta-$projectname.classif $outputclassif
+fi
+
+
+workingconfigfile=$working_dir/TEdenovo_Galaxy_config_$projectname_complete
+sed -i 's|repet_host:.*|repet_host:|g' $workingconfigfile
+sed -i 's|repet_user:.*|repet_user:|g' $workingconfigfile
+sed -i 's|repet_pw:.*|repet_pw:|g' $workingconfigfile
+sed -i 's|repet_db:.*|repet_db:|g' $workingconfigfile
+sed -i 's|repet_port:.*|repet_port:|g' $workingconfigfile
+sed -i 's|repet_job_manager:.*|repet_job_manager:|g' $workingconfigfile
+sed -i 's|project_name:.*|project_name: '$projectname'|g' $workingconfigfile
+sed -i 's|project_dir:.*|project_dir:|g' $workingconfigfile
+sed -i 's|tmpDir:.*|tmpDir:|g' $workingconfigfile
+mv $workingconfigfile $outputconfig
+

diff -r 000000000000 -r baea09e6722b TEdenovo.xml
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/TEdenovo.xml Mon Feb 06 13:31:53 2017 -0500

[

b'@@ -0,0 +1,322 @@\n+<tool id="tedenovo" name="REPET Lite - TEdenovo" version="2.2.0">\n+ \n+ \n+ <description> Compute a library of transposable element</description>\n+ \n+ \n+ <requirements>\n+ <requirement type="binary">python</requirement>\n+ <requirement type="package" version="2.5">repet</requirement>\n+ </requirements>\n+ \t \n+ \n+ <stdio>\n+ \n+ <exit_code range="1:" level="fatal"/>\n+ <exit_code range=":-1" level="fatal"/>\n+\n+ </stdio>\n+\n+ \n+ <version_command>\n+ \tTEdenovo.py --version\n+ </version_command>\n+ \n+ \n+ <command interpreter="bash">\n+ \tTEdenovo.sh $fasta $outputfasta $classif $outputlog $outputconfig $outputstats\n+ \t#if str( $classif ) == "yes":\n+ \t $outputclassif\n+ \t#else\n+ \t\'\'\n+ \t#end if\n+ </command>\n+\t \n+ \n+ <inputs>\n+ \t<param name="fasta" type="data" format="fasta" optional="false" label="Fasta alignment input" />\n+ \t<param name="classif" type="select" label="Get classification informations" help="To add the informations at annotation file on next step." >\n+ \t\t<option value="no" selected="true">No</option>\n+ \t <option value="yes">Yes</option>\n+ \t</param>\n+ \t<param name="label" type="text" label="Output name" />\n+ </inputs> \n+ \t\t\t\t \t \t \t \t \t \t \t \t \n+ \n+ <outputs>\n+ <data name="outputlog" type="data" format="txt" label="TEdenovo-#if str($label)==\'\' then $fasta.name else $label #.log" />\n+ \t<data name="outputfasta" type="data" format="fasta" label="TEdenovo-#if str($label)==\'\' then $fasta.name else $label #.TElib.fa" />\n+ \t<data name="outputstats" type="data" format="txt" label="TEdenovo-#if str($label)==\'\' then $fasta.name else $label #.classif_stats.txt" />\n+ \t<data name="outputclassif" type="data" format="tabular" label="TEdenovo-#if str($label)==\'\' then $fasta.name else $label #.classif" >\n+ \t\t<filter>(classif == \'yes\')</filter>\n+ \t</data>\n+ \t<data name="outputconfig" type="data" format="txt" label="TEdenovo-#if str($label)==\'\' then $fasta.name else $label #.cfg" />\n+ </outputs>\n+ \t\t \n+ \n+ \n+ <tests>\n+ \n+ <test>\n+ <param name="fasta" value="DmelChr4Chr3.fa" />\n+ <output name="outputfasta">\n+ \t<assert_contents>\n+ \t\t<has_line_matching expression="^>\\w+" />\n+ \t\t<has_line_matching expression="[ACTG]{60}" />\n+ \t</assert_contents>\n+ </output>\n+ <output name="outputlog">\n+ <assert_contents>\n+ <has_line_matching expression="^step 7 finished successfully" />\n+ <has_line_matching expression="^END time: \\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}" />\n+\t\t\t<has_line_matching expression="^Writing fasta file" />\n+ </assert_contents>\n+ </output>\n+ <output name="outputconfig">\n+ \t<assert_contents>\n+ \t\t<has_line_matching expression="^project_name: \\d{8}" />\n+ \t\t<has_line_matching expression="^repet_version: 2.5" />\n+ \t\t<has_line_matching expression="^tmpDir:" />\n+ \t\t<has_line_matching expression="^clean: yes" />\n+ \t</assert_contents>\n+ </output>\n+ <output name="outputstats">\n+ \t<assert_cont'..b' = {PLoS ONE},\n+\t publisher = {Public Library of Science},\n+\t title = {Considering Transposable Element Diversification in <italic>De Novo</italic> Annotation Approaches},\n+\t year = {2011},\n+\t month = {01},\n+\t volume = {6},\n+\t url = {http://dx.doi.org/10.1371%2Fjournal.pone.0016526},\n+\t pages = {e16526},\n+\t abstract = {\n+\t <p>Transposable elements (TEs) are mobile, repetitive DNA sequences that are almost ubiquitous in prokaryotic and eukaryotic genomes. They have a large impact on genome structure, function and evolution. With the recent development of high-throughput sequencing methods, many genome sequences have become available, making possible comparative studies of TE dynamics at an unprecedented scale. Several methods have been proposed for the <italic>de novo</italic> identification of TEs in sequenced genomes. Most begin with the detection of genomic repeats, but the subsequent steps for defining TE families differ. High-quality TE annotations are available for the <italic>Drosophila melanogaster</italic> and <italic>Arabidopsis thaliana</italic> genome sequences, providing a solid basis for the benchmarking of such methods. We compared the performance of specific algorithms for the clustering of interspersed repeats and found that only a particular combination of algorithms detected TE families with good recovery of the reference sequences. We then applied a new procedure for reconciling the different clustering results and classifying TE sequences. The whole approach was implemented in a pipeline using the REPET package. Finally, we show that our combined approach highlights the dynamics of well defined TE families by making it possible to identify structural variations among their copies. This approach makes it possible to annotate TE families and to study their diversification in a single analysis, improving our understanding of TE dynamics at the whole-genome scale and for diverse species.</p>\n+\t },\n+\t number = {1},\n+\t doi = {10.1371/journal.pone.0016526}\n+\t}]]></citation>\n+\t<citation type="bibtex"><![CDATA[@article{10.1371/journal.pone.0094101,\n+ author = {Maumus, Florian AND Quesneville, Hadi},\n+ journal = {PLoS ONE},\n+ publisher = {Public Library of Science},\n+ title = {Deep Investigation of <italic>Arabidopsis thaliana</italic> Junk DNA Reveals a Continuum between Repetitive Elements and Genomic Dark Matter},\n+ year = {2014},\n+ month = {04},\n+ volume = {9},\n+ url = {http://dx.doi.org/10.1371%2Fjournal.pone.0094101},\n+ pages = {e94101},\n+ abstract = {<p>Eukaryotic genomes contain highly variable amounts of DNA with no apparent function. This so-called junk DNA is composed of two components: repeated and repeat-derived sequences (together referred to as the repeatome), and non-annotated sequences also known as genomic dark matter. Because of their high duplication rates as compared to other genomic features, transposable elements are predominant contributors to the repeatome and the products of their decay is thought to be a major source of genomic dark matter. Determining the origin and composition of junk DNA is thus important to help understanding genome evolution as well as host biology. In this study, we have used a combination of tools enabling to show that the repeatome from the small and reducing <italic>A. thaliana</italic> genome is significantly larger than previously thought. Furthermore, we present the concepts and results from a series of innovative approaches suggesting that a significant amount of the <italic>A. thaliana</italic> dark matter is of repetitive origin. As a tentative standard for the community, we propose a deep compendium annotation of the <italic>A. thaliana</italic> repeatome that may help addressing farther genome evolution as well as transcriptional and epigenetic regulation in this model plant.</p>},\n+ number = {4},\n+ doi = {10.1371/journal.pone.0094101}\n+\t}]]></citation>\n+ </citations>\n+\n+ \n+</tool>\n'

diff -r 000000000000 -r baea09e6722b TEdenovo_lite.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/TEdenovo_lite.py Mon Feb 06 13:31:53 2017 -0500

[

b'@@ -0,0 +1,406 @@\n+#!/usr/bin/env python\n+\n+\n+import os\n+import sys\n+import time\n+import glob\n+import shutil\n+import ConfigParser\n+from commons.core.seq.FastaUtils import *\n+import operator\n+import re\n+\n+\n+\n+if not "REPET_PATH" in os.environ.keys():\n+ print "ERROR: no environment variable REPET_PATH"\n+ sys.exit(1)\n+\n+if (not "REPET_DB" in os.environ.keys()) or (not "REPET_HOST" in os.environ.keys()) or (not "REPET_PORT" in os.environ.keys()) or (not "REPET_USER" in os.environ.keys()) or (not "REPET_PW" in os.environ.keys()):\n+ print "ERROR: there is at least one environment database variable missing : REPET_DB, REPET_PORT, REPET_HOST, REPET_USER or REPET_PW"\n+ sys.exit(1)\n+\n+if not "REPET_JOB_MANAGER" in os.environ.keys():\n+ print "ERROR: no environment variable REPET_JOB_MANAGER"\n+ sys.exit(1)\n+\n+\n+if not "%s/bin" % os.environ["REPET_PATH"] in os.environ["PATH"]: \n+ os.environ["PATH"] = "%s/bin:%s" % (os.environ["REPET_PATH"], os.environ["PATH"])\n+ \n+sys.path.append(os.environ["REPET_PATH"])\n+if not "PYTHONPATH" in os.environ.keys():\n+ os.environ["PYTHONPATH"] = os.environ["REPET_PATH"]\n+else:\n+ os.environ["PYTHONPATH"] = "%s:%s" % (os.environ["REPET_PATH"], os.environ["PYTHONPATH"])\n+\n+from commons.core.LoggerFactory import LoggerFactory\n+from commons.core.checker.RepetException import RepetException\n+from commons.core.utils.FileUtils import FileUtils\n+from commons.core.utils.RepetOptionParser import RepetOptionParser\n+from commons.core.seq.FastaUtils import FastaUtils\n+from commons.core.sql.DbFactory import DbFactory\n+from itertools import islice\n+\n+LOG_DEPTH = "TEdenovo.pipeline"\n+\n+class TEdenovo_lite(object):\n+\n+ def __init__(self, configFileName = "", fastaFileName = "", verbosity = 0):\n+ self._configFileName = configFileName\n+ self._fastaFileName = os.path.abspath(fastaFileName)\n+ self._projectName = time.strftime("%Y%m%d%H%M%S")\n+ self._limitSeqSize = 200000000\n+\n+\tif "REPET_NUCL_BANK" in os.environ.keys():\n+\t if os.path.exists(os.environ["REPET_NUCL_BANK"]):\n+ \tself._nucl_bank = os.environ["REPET_NUCL_BANK"]\n+\t else : \n+\t\tprint "ERROR : the nucleotides bank configured doesn\'t exist. Please correct it in the REPET_NUCL_BANK variable"\n+ sys.exit(1)\n+\telse : \n+\t self._nucl_bank = ""\n+\tif "REPET_PROT_BANK" in os.environ.keys():\n+ if os.path.exists(os.environ["REPET_PROT_BANK"]):\n+\t\tself._prot_bank = os.environ["REPET_PROT_BANK"]\n+\t else :\n+ print "ERROR : the proteins bank configured doesn\'t exist. Please correct it in the REPET_PROT_BANK variable"\n+ sys.exit(1)\n+\telse :\n+\t self._prot_bank = ""\n+\tif "REPET_HMM_PROFILES" in os.environ.keys():\n+ if os.path.exists(os.environ["REPET_HMM_PROFILES"]):\n+ \tself._HMM_profiles = os.environ["REPET_HMM_PROFILES"]\n+ else :\n+ print "ERROR : the hmm profiles bank configured doesn\'t exist. Please correct it in the REPET_HMM_PROFILES variable"\n+ sys.exit(1)\n+\telse : \n+\t self._HMM_profiles = ""\n+\tif "REPET_RDNA_BANK" in os.environ.keys():\n+ if os.path.exists(os.environ["REPET_RDNA_BANK"]):\n+ \tself._rdna_bank = os.environ["REPET_RDNA_BANK"]\n+ else :\n+ print "ERROR : the rDNA bank configured doesn\'t exist. Please correct it in the REPET_PROT_BANK variable"\n+ sys.exit(1)\n+\telse :\n+ self._rdna_bank = ""\n+\tif self._nucl_bank == "" and self._prot_bank == "" and self._HMM_profiles == "" and self._rdna_bank == "" : \n+\t print "WARNING : No bank are configured ... To set banks please add REPET_NUCL_BANK, REPET_PROT_BANK, REPET_HMM_PROFILES and/or REPET_RDNA_BANK in your environment"\n+\tif "REPET_TMP_DIR" in os.environ.keys():\n+\t self._tmp_dir = os.environ["REPET_TMP_DIR"]\n+\telse : \n+\t self._tmp_dir = ""\n+ self._outputFasta = ""\n+ self._classif = False\n+ self._outputClassif = ""\n+\tself._outputStats = "'..b'.py -P %s -C %s -S 6 -s Blaster -c GrpRecPil -m Map -v %i" % (self._projectName, self._configFileName, self._verbosity) )\n+ lCmds.append( "TEdenovo.py -P %s -C %s -S 7 -s Blaster -c GrpRecPil -m Map -v %i" % (self._projectName, self._configFileName, self._verbosity) )\n+\n+ for cmd in lCmds:\n+ returnValue = os.system(cmd)\n+ if returnValue != 0:\n+ print "ERROR: command \'%s\' returned %i" % (cmd, returnValue)\n+ self._cleanTables()\n+ sys.exit(1)\n+\n+ print "END time: %s" % time.strftime("%Y-%m-%d %H:%M:%S")\n+ outFastaFile = glob.glob("%s_Blaster_GrpRecPil_Map_TEclassif_Filtered/*_denovoLibTEs_filtered.fa"%self._projectName) \n+ shutil.copy(outFastaFile[0], self._outputFasta)\n+\toutStatsFile = glob.glob("%s_Blaster_GrpRecPil_Map_TEclassif_Filtered/*.classif_stats.txt"%self._projectName)\n+ shutil.copy(outStatsFile[0], self._outputStats)\n+ if self._classif:\n+ outClassifFile = glob.glob("%s_Blaster_GrpRecPil_Map_TEclassif/classifConsensus/*_withoutRedundancy_negStrandReversed_WickerH.classif"%self._projectName)\n+ shutil.copy(outClassifFile[0], self._outputClassif)\n+ self._renameTE()\n+\n+ def _renameTE(self):\n+ name=re.search(r\'([^\\/\\\\]*)-\\d{8}-denovoLibTEs_filtered\\.[fa|fasta|fsa|fas]\',self._outputFasta).groups()[0]\n+ os.system("sed -i \'s|%s|%s|\' %s" % (self._projectName,name, self._outputFasta))\n+ if self._classif:\n+ os.system("sed -i \'s|%s|%s|\' %s" % (self._projectName,name, self._outputClassif))\t\n+\n+ def preprocessFastaFile(self):\n+ inFileHandler = open(self._fastaFileName, "r")\n+ cumulLength = FastaUtils.dbCumLength(inFileHandler)\n+ inFileHandler.close()\n+ if cumulLength >= self._limitSeqSize:\n+ print "Preprocess lauched"\n+ allContigsBSDB=self.removeNstretches()\n+ selectedContigsBSDB=self.selectContigs4givenSize(allContigsBSDB)\n+ self.correctHeader(selectedContigsBSDB)\n+ fastaFile=self.writeFastaInput(selectedContigsBSDB)\n+ print "Preprocess finished"\n+ else:\n+ fastaFile=self._fastaFileName\n+ print "No preprocess : the genome size %s lower than %s Mbp" % (cumulLength, self._limitSeqSize/1000000)\n+ os.symlink(fastaFile,"%s/%s.fa" %(os.getcwd(),self._projectName)) #creer repertoire projet\n+\n+ def _launchListAndDropTables(self):\t\n+ cmd = "ListAndDropTables.py"\n+ cmd += " -C %s" % self._configFileName\n+ cmd += " -d \'%s\'" % self._projectName\n+ os.system(cmd)\n+\n+ def _cleanJobsTable(self):\n+ db = DbFactory.createInstance( configFileName = self._configFileName )\n+ sql_cmd="DELETE FROM jobs WHERE groupid like \'%s%%\';"%self._projectName\n+ db.execute( sql_cmd )\n+ db.close()\n+\n+ def _cleanTables(self):\n+ self._launchListAndDropTables()\n+ self. _cleanJobsTable()\n+\n+\n+ def run(self):\n+ os.mkdir(self._projectName)\n+ os.chdir(self._projectName)\n+ self._writeConfigFile()\n+ self.preprocessFastaFile()\n+ if self._nucl_bank != "" and self._nucl_bank != None:\n+ os.symlink(self._nucl_bank,"%s/%s" %(os.getcwd(),os.path.basename(self._nucl_bank)))\n+ if self._prot_bank != "" and self._prot_bank != None:\n+ os.symlink(self._prot_bank,"%s/%s" %(os.getcwd(),os.path.basename(self._prot_bank)))\n+ if self._HMM_profiles != "" and self._HMM_profiles != None:\n+ os.symlink(self._HMM_profiles,"%s/%s" %(os.getcwd(),os.path.basename(self._HMM_profiles)))\n+ if self._rdna_bank != "" and self._rdna_bank != None:\n+ os.symlink(self._rdna_bank,"%s/%s" %(os.getcwd(),os.path.basename(self._rdna_bank)))\n+\n+ self._launchTEdenovo()\n+ self._cleanTables()\n+\n+if __name__ == \'__main__\':\n+ iTEdenovo = TEdenovo_lite()\n+ iTEdenovo.setAttributesFromCommandLine()\n+ iTEdenovo.run()\n'

diff -r 000000000000 -r baea09e6722b tool_dependencies.xml
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/tool_dependencies.xml Mon Feb 06 13:31:53 2017 -0500

@@ -0,0 +1,6 @@
+<?xml version="1.0"?>
+<tool_dependency>
+ <package name="repet" version="2.5">
+ <repository changeset_revision="f3b6009634b1" name="package_repet_2_5" owner="vmarcon" toolshed="https://toolshed.g2.bx.psu.edu" />
+ </package>
+</tool_dependency>