Mercurial > repos > greg > plant_tribes_assembly_post_processor

changeset 4:b0d9dc3c8535 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Uploaded
author greg
date Mon, 30 Oct 2017 09:51:13 -0400
parents 07ee186ba16e
children 2732db7dd0ef
files .shed.yml assembly_post_processor.py assembly_post_processor.xml macros.xml test-data/output.pttgf test-data/output_targeted_gene_families_stats.tabular test-data/tool-data/plant_tribes/scaffolds/README.txt utils.py
diffstat 8 files changed, 50 insertions(+), 222 deletions(-) [+]
line wrap: on
line diff
--- a/.shed.yml	Thu Aug 24 13:10:29 2017 -0400
+++ b/.shed.yml	Mon Oct 30 09:51:13 2017 -0400
@@ -9,7 +9,7 @@
   utilize objective classifications of complete protein sequences from sequenced plant genomes to perform
   comparative evolutionary studies. It postprocesses de novo assembly transcripts into putative coding
   sequences and their corresponding amino acid translations, locally assembling targeted gene families.
-remote_repository_url: https://github.com/gregvonkuster/galaxy_tools/tree/master/tools/plant_tribes/assembly_post_processor
+remote_repository_url: https://github.com/gregvonkuster/galaxy_tools/tree/master/tools/phylogenetics/plant_tribes/assembly_post_processor
 type: unrestricted
 categories:
 - Phylogenetics
--- a/assembly_post_processor.py	Thu Aug 24 13:10:29 2017 -0400
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,69 +0,0 @@
-#!/usr/bin/env python
-import argparse
-import os
-import shutil
-
-import utils
-
-OUTPUT_DIR = 'assemblyPostProcessing_dir'
-
-parser = argparse.ArgumentParser()
-parser.add_argument('--dereplicate', dest='dereplicate', default=None, help='Remove duplicate sequences')
-parser.add_argument('--gap_trimming', dest='gap_trimming', type=float, default=0, help='Trim alignments')
-parser.add_argument('--gene_family_search', dest='gene_family_search', default=None, help='Targeted gene families')
-parser.add_argument('--method', dest='method', default=None, help='Protein clustering method')
-parser.add_argument('--min_length', dest='min_length', type=int, default=0, help='Minimum sequence length')
-parser.add_argument('--num_threads', dest='num_threads', type=int, help='Number of processors')
-parser.add_argument('--output_pttgf', dest='output_pttgf', default=None, help='Primary targeted gene families dataset')
-parser.add_argument('--output_cds', dest='output_cds', help='Output transcripts.cds')
-parser.add_argument('--output_cleaned_cds', dest='output_cleaned_cds', help='Output transcripts.cleaned.cds')
-parser.add_argument('--output_cleaned_nr_cds', dest='output_cleaned_nr_cds', default=None, help='Output transcripts.cleaned.nr.cds')
-parser.add_argument('--output_cleaned_nr_pep', dest='output_cleaned_nr_pep', default=None, help='Output transcripts.cleaned.nr.pep')
-parser.add_argument('--output_cleaned_pep', dest='output_cleaned_pep', help='Output transcripts.cleaned.pep')
-parser.add_argument('--output_pep', dest='output_pep', help='Output transcripts.pep')
-parser.add_argument('--output_pttgf_dir', dest='output_pttgf_dir', default=None, help='Directory hierarchy of targeted gene family datasets')
-parser.add_argument('--prediction_method', dest='prediction_method', help='Coding regions prediction method')
-parser.add_argument('--scaffold', dest='scaffold', default=None, help='Gene family scaffold')
-parser.add_argument('--score_matrices', dest='score_matrices', default=None, help='Scores matrices')
-parser.add_argument('--strand_specific', dest='strand_specific', default=None, help='Strand-specific assembly')
-parser.add_argument('--transcripts', dest='transcripts', help='Transcriptome assembly fasta file')
-
-args = parser.parse_args()
-
-# Build the command line.
-cmd = 'AssemblyPostProcessor'
-if args.dereplicate is not None:
-    cmd += ' --dereplicate'
-if args.gap_trimming > 0:
-    cmd += ' --gap_trimming %4f' % args.gap_trimming
-if args.gene_family_search is not None:
-    cmd += ' --gene_family_search %s' % args.gene_family_search
-if args.method is not None:
-    cmd += ' --method %s' % args.method
-if args.min_length > 0:
-    cmd += ' --min_length %d' % args.min_length
-cmd += ' --num_threads %d' % args.num_threads
-cmd += ' --prediction_method %s' % args.prediction_method
-if args.scaffold is not None:
-    cmd += ' --scaffold %s' % args.scaffold
-if args.score_matrices is not None:
-    cmd += ' --score_matrices %s' % args.score_matrices
-if args.strand_specific is not None:
-    cmd += ' --strand_specific'
-cmd += ' --transcripts %s' % args.transcripts
-# Run the command.
-utils.run_command(cmd)
-
-# Handle outputs.
-shutil.move(os.path.join(OUTPUT_DIR, 'transcripts.cds'), args.output_cds)
-shutil.move(os.path.join(OUTPUT_DIR, 'transcripts.cleaned.cds'), args.output_cleaned_cds)
-if args.output_cleaned_nr_cds is not None:
-    shutil.move(os.path.join(OUTPUT_DIR, 'transcripts.cleaned.nr.cds'), args.output_cleaned_nr_cds)
-if args.output_cleaned_nr_pep is not None:
-    shutil.move(os.path.join(OUTPUT_DIR, 'transcripts.cleaned.nr.pep'), args.output_cleaned_nr_pep)
-shutil.move(os.path.join(OUTPUT_DIR, 'transcripts.cleaned.pep'), args.output_cleaned_pep)
-shutil.move(os.path.join(OUTPUT_DIR, 'transcripts.pep'), args.output_pep)
-if args.output_pttgf is not None and args.output_pttgf_dir is not None:
-    src_output_dir = os.path.join(OUTPUT_DIR, 'targeted_gene_families')
-    utils.move_directory_files(src_output_dir, args.output_pttgf_dir)
-    utils.write_html_output(args.output_pttgf, 'Targeted gene families', args.output_pttgf_dir)
\ No newline at end of file
--- a/assembly_post_processor.xml	Thu Aug 24 13:10:29 2017 -0400
+++ b/assembly_post_processor.xml	Mon Oct 30 09:51:13 2017 -0400
@@ -1,11 +1,14 @@
-<tool id="plant_tribes_assembly_post_processor" name="AssemblyPostProcessor" version="@WRAPPER_VERSION@.2">
+<tool id="plant_tribes_assembly_post_processor" name="AssemblyPostProcessor" version="@WRAPPER_VERSION@.3.0">
     <description>post-processes de novo transcriptome assembly</description>
     <macros>
         <import>macros.xml</import>
     </macros>
-    <expand macro="requirements_assembly_post_processor" />
+    <requirements>
+        <requirement type="package" version="1.0.3">plant_tribes_assembly_post_processor</requirement>
+    </requirements>
     <command detect_errors="exit_code"><![CDATA[
-python '$__tool_directory__/assembly_post_processor.py'
+#set output_dir = 'assemblyPostProcessing_dir'
+AssemblyPostProcessor
 --transcripts '$input'
 --prediction_method $prediction_method_cond.prediction_method
 #if str($prediction_method_cond.prediction_method) == 'estscan':
@@ -15,27 +18,36 @@
     #set target_gene_family_assembly_cond = $options_type.target_gene_family_assembly_cond
     #if str($target_gene_family_assembly_cond.target_gene_family_assembly) == 'yes':
         --gene_family_search '$target_gene_family_assembly_cond.orthogroups'
-        --output_pttgf '$output_pttgf'
-        --output_pttgf_dir '$output_pttgf.files_path'
         --scaffold '$target_gene_family_assembly_cond.scaffold.fields.path'
         --method '$target_gene_family_assembly_cond.method'
         --gap_trimming $target_gene_family_assembly_cond.gap_trimming
+        #if str($target_gene_family_assembly_cond.min_coverage) != '0.0':
+            --min_coverage $target_gene_family_assembly_cond.min_coverage
+        #end if
     #end if
     #if str($options_type.strand_specific) == 'yes':
-        --strand_specific true
+        --strand_specific
     #end if
     #if str($options_type.dereplicate) == 'yes':
-        --dereplicate true
-        --output_cleaned_nr_cds '$output_cleaned_nr_cds'
-        --output_cleaned_nr_pep '$output_cleaned_nr_pep'
+        --dereplicate
     #end if
     --min_length $options_type.min_length
 #end if
 --num_threads \${GALAXY_SLOTS:-4}
---output_cds '$output_cds'
---output_cleaned_cds '$output_cleaned_cds'
---output_cleaned_pep '$output_cleaned_pep'
---output_pep '$output_pep'
+&>proc.log
+&& mv $output_dir/transcripts.cds '$output_cds'
+&& mv $output_dir/transcripts.pep '$output_pep'
+&& mv $output_dir/transcripts.cleaned.cds '$output_cleaned_cds'
+&& mv $output_dir/transcripts.cleaned.pep '$output_cleaned_pep'
+#if str($options_type.options_type_selector) == 'advanced':
+    #if str($target_gene_family_assembly_cond.target_gene_family_assembly) == 'yes':
+        && mv $output_dir/targeted_gene_family_assemblies.stats '$output_targeted_gene_families_stats'
+    #end if
+    #if str($options_type.dereplicate) == 'yes':
+        && mv $output_dir/transcripts.cleaned.nr.cds '$output_cleaned_nr_cds'
+        && mv $output_dir/transcripts.cleaned.nr.pep '$output_cleaned_nr_pep'
+    #end if
+#end if
     ]]></command>
     <inputs>
         <param name="input" format="fasta" type="data" label="Transcriptome assembly fasta file"/>
@@ -74,6 +86,7 @@
                             <option value="orthomcl">OrthoMCL</option>
                         </param>
                         <param name="gap_trimming" type="float" value="0.1" min="0" max="1.0" label="Trim alignments"/>
+                        <param name="min_coverage" type="float" value="0" min="0" max="1.0" label="Minimum alignment coverage"/>
                     </when>
                 </conditional>
                 <param name="strand_specific" type="select" label="Strand-specific assembly?">
@@ -89,9 +102,13 @@
         </conditional>
     </inputs>
     <outputs>
-        <data name="output_pttgf" format="pttgf" label="Targeted gene families: ${tool.name} on ${on_string}">
+        <data name="output_targeted_gene_families_stats" format="tabular" label="Targeted gene families statistics: ${tool.name} on ${on_string}">
             <filter>options_type['options_type_selector'] == 'advanced' and options_type['target_gene_family_assembly_cond']['target_gene_family_assembly'] == 'yes'</filter>
         </data>
+        <collection name="output_targeted_gene_families" type="list" label="Targeted gene families: ${tool.name} on ${on_string}">
+            <discover_datasets pattern="__name__" directory="assemblyPostProcessing_dir/targeted_gene_family_assemblies" format="fasta" />
+            <filter>options_type['options_type_selector'] == 'advanced' and options_type['target_gene_family_assembly_cond']['target_gene_family_assembly'] == 'yes'</filter>
+        </collection>
         <data name="output_pep" format="fasta" label="transcripts.pep: ${tool.name} on ${on_string}"/>
         <data name="output_cleaned_pep" format="fasta" label="transcripts.cleaned.pep: ${tool.name} on ${on_string}"/>
         <data name="output_cleaned_nr_pep" format="fasta" label="transcripts.cleaned.nr.pep: ${tool.name} on ${on_string}">
@@ -134,7 +151,9 @@
             <param name="scaffold" value="22Gv1.1"/>
             <param name="method" value="orthomcl"/>
             <param name="dereplicate" value="yes"/>
-            <output name="output_pttgf" file="output.pttgf" ftype="pttgf"/>
+            <output_collection name="output_targeted_gene_families" type="list">
+            </output_collection>
+            <output name="output_targeted_gene_families_stats" file="output_targeted_gene_families_stats.tabular" ftype="tabular"/>
             <output name="output_cds" file="transcripts_tgf.cds" ftype="fasta"/>
             <output name="output_cleaned_cds" file="transcripts.cleaned_tgf.cds" ftype="fasta"/>
             <output name="output_cleaned_nr_cds" file="transcripts_tgf.cleaned.nr.cds" ftype="fasta"/>
@@ -163,10 +182,11 @@
 
  * **Perform targeted gene assembly?** - selecting 'Yes' enables local assembly of one or more targeted gene families in a specific scaffold.  Scaffolds are defined in PlantTribes as clusters of paralogous/orthologous sequences from a specified set of proteomes[5-7].
 
-   * **Targeted gene families** - select a history item containing a list of targeted orthogroup identifiers corresponding to the gene family classification from a specified scaffold.  Gene family identifiers can be obtained from the function annotation table ("Orthogroup ID" field of .summary file) of scaffold data installed into Galaxy via the PlantTribes Scaffolds Download Data Manager tool, and also available at the PlantTribes github repository (https://github.com/dePamphilis/PlantTribes/tree/master/config).
+   * **Targeted gene families** - select a history item containing a list of targeted orthogroup identifiers corresponding to the gene family classification from a specified scaffold. Gene family identifiers can be obtained from the function annotation table ("Orthogroup ID" field of .summary file) of scaffold data installed into Galaxy via the PlantTribes Scaffolds Download Data Manager tool, and are also available in the PlantTribes "annotation" directory of the scaffold data download.
    * **Gene family scaffold** - one of the PlantTribes gene family scaffolds (installed into Galaxy by the PlantTribes Scaffolds Download Data Manager tool) whose orthogroup(s) are targeted for the localized assembly.
    * **Protein clustering method** - gene family scaffold protein clustering method.  Each PlantTribes scaffold data has up to three sets of clusters - GFam[8] (clusters of consensus domain architecture), OrthoFinder[9] (broadly defined clusters) or OrthoMCL[10] (narrowly defined clusters).  You can also install your own data scaffold created using a different clustering method as long as it conforms to the PlantTribes scaffold data format.
    * **Trim alignments** - trim gene family multiple sequence alignments that include scaffold backbone genes and locally assembled transcripts to remove non-conserved regions (gappy sites)[11].  The trimmed alignments are used in assigning scores to locally assembled transcripts to determine how well they compare to the backbone gene models.  The default setting of 0.1 removes sites that have gaps in 90% or more of the sequences in the multiple sequence alignment.  This option is restricted to the range 0.0 - 1.0.
+   * **Minimum alignment coverage** - allowable sequence coverage in the orthogroup trimmed protein multiple sequence alignments.  Selecting transcripts with coverage of at least the average of the backbone orthogroup gene models is recommended.  Details are shown in the targeted gene family assembly statistics history item.
 
  * **Strand-specific assembly?** - select 'Yes' if transcriptome library sequences were strand-specific.  If 'Yes" is selected, transcripts from the minority strand (antisense) are removed.
  * **Remove duplicate sequences?** - select 'Yes' to remove duplicated and exact subsequences[12].
--- a/macros.xml	Thu Aug 24 13:10:29 2017 -0400
+++ b/macros.xml	Mon Oct 30 09:51:13 2017 -0400
@@ -1,47 +1,6 @@
 <?xml version='1.0' encoding='UTF-8'?>
 <macros>
     <token name="@WRAPPER_VERSION@">1.0</token>
-    <xml name="requirements_assembly_post_processor">
-        <requirements>
-            <requirement type="package" version="1.0.2">plant_tribes_assembly_post_processor</requirement>
-        </requirements>
-    </xml>
-    <xml name="requirements_gene_family_aligner">
-        <requirements>
-            <requirement type="package" version="1.0.2">plant_tribes_gene_family_aligner</requirement>
-        </requirements>
-    </xml>
-    <xml name="requirements_gene_family_classifier">
-        <requirements>
-            <requirement type="package" version="1.0.2">plant_tribes_gene_family_classifier</requirement>
-        </requirements>
-    </xml>
-    <xml name="requirements_gene_family_integrator">
-        <requirements>
-            <requirement type="package" version="1.0.2">plant_tribes_gene_family_integrator</requirement>
-        </requirements>
-    </xml>
-    <xml name="requirements_kaks_analysis">
-        <requirements>
-            <requirement type="package" version="1.0.2">plant_tribes_kaks_analysis</requirement>
-        </requirements>
-    </xml>
-    <xml name="requirements_ks_distribution">
-        <requirements>
-            <requirement type="package" version="1.3.2">r-optparse</requirement>
-        </requirements>
-    </xml>
-    <xml name="requirements_gene_family_phylogeny_builder">
-        <requirements>
-            <requirement type="package" version="1.0.2">plant_tribes_gene_family_phylogeny_builder</requirement>
-        </requirements>
-    </xml>
-    <xml name="param_codon_alignments">
-        <param name="codon_alignments" type="select" label="Codon alignments">
-            <option value="yes" selected="true">Yes</option>
-            <option value="no">No</option>
-        </param>
-    </xml>
     <xml name="param_method">
         <param name="method" type="select" label="Protein clustering method">
             <option value="gfam" selected="true">GFam</option>
@@ -49,74 +8,12 @@
             <option value="orthomcl">OrthoMCL</option>
         </param>
     </xml>
-    <xml name="param_options_type">
-        <param name="options_type" type="select" label="Options Configuration">
-            <option value="basic" selected="true">Basic</option>
-            <option value="advanced">Advanced</option>
-        </param>
-    </xml>
-    <xml name="param_orthogroup_fna">
-        <param name="orthogroup_fna" type="select" label="Orthogroups coding sequences">
-            <option value="yes" selected="true">Yes</option>
-            <option value="no">No</option>
-        </param>
-    </xml>
     <xml name="param_scaffold">
         <param name="scaffold" type="select" label="Gene family scaffold">
             <options from_data_table="plant_tribes_scaffolds" />
             <validator type="no_options" message="No PlantTribes scaffolds are available.  Use the PlantTribes Scaffolds Download Data Manager tool in Galaxy to install and populate the PlantTribes scaffolds data table." />
         </param>
     </xml>
-    <xml name="param_sequence_type">
-        <param name="sequence_type" type="select" label="Sequence type used in the phylogenetic inference (dna)">
-            <option value="protein" selected="true">Amino acid based</option>
-            <option value="dna">Nucleotide based</option>
-        </param>
-    </xml>
-    <xml name="cond_alignment_method">
-        <conditional name="alignment_method_cond">
-            <param name="alignment_method" type="select" force_select="true" label="Multiple sequence alignment method">
-                <option value="mafft" selected="true">MAFFT</option>
-                <option value="pasta">PASTA</option>
-            </param>
-            <when value="mafft" />
-            <when value="pasta">
-                <param name="pasta_iter_limit" type="integer" value="3" min="1" label="PASTA iteration limit" />
-            </when>
-        </conditional>
-    </xml>
-    <xml name="cond_remove_gappy_sequences">
-        <conditional name="remove_gappy_sequences_cond">
-            <param name="remove_gappy_sequences" type="select" label="Alignment post-processing configuration">
-                <option value="no" selected="true">No</option>
-                <option value="yes">Yes</option>
-            </param>
-            <when value="no" />
-            <when value="yes">
-                <conditional name="trim_type_cond">
-                    <param name="trim_type" type="select" label="Trimming method">
-                        <option value="gap_trimming" selected="true">Gap score based trimming</option>
-                        <option value="automated_trimming">Automated heuristic trimming</option>
-                    </param>
-                    <when value="gap_trimming">
-                        <param name="gap_trimming" type="float" optional="true" min="0" max="1.0" label="Gap score" />
-                    </when>
-                    <when value="automated_trimming" />
-                </conditional>
-                <conditional name="remove_sequences_with_gaps_cond">
-                    <param name="remove_sequences_with_gaps" type="select" label="Remove sequences">
-                        <option value="no" selected="true">No</option>
-                        <option value="yes">Yes</option>
-                    </param>
-                    <when value="no" />
-                    <when value="yes">
-                        <param name="remove_sequences_with_gaps_of" type="float" optional="true" min="0" max="1" label="Coverage score" />
-                        <param name="iterative_realignment" type="integer" optional="true" min="0" label="Realignment iteration limit" />
-                    </when>
-                </conditional>
-            </when>
-        </conditional>
-    </xml>
     <xml name="citation1">
         <citation type="bibtex">
             @misc{None,
--- a/test-data/output.pttgf	Thu Aug 24 13:10:29 2017 -0400
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,6 +0,0 @@
-<html><head><h3>Targeted gene families: 1 items</h3></head>
-<body><p/><table cellpadding="2">
-<tr><b>Directories</th></b>
-<tr bgcolor="#D8D8D8"><td><a href="752" type="text/plain">752</a>
-</td></tr>
-</table></body></html>
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test-data/output_targeted_gene_families_stats.tabular	Mon Oct 30 09:51:13 2017 -0400
@@ -0,0 +1,7 @@
+# seq_id = sequence identity
+# cov = sequence coverage in the orthogroup trimmed protein alignment
+# avg_cov = mean coverage of orthogroup backbone sequences in the trimmed protein alignment
+# sd_cov = standard deviation (for coverage mean) of orthogroup backbone sequences in the trimmed protein alignment
+# len = total-length of conserved regions of sequence in the orthogroup trimmed protein alignment
+# avg_len = mean total-lengths of conserved regions of backbone sequences in the trimmed orthogroup protein alignment
+# sd_len = standard deviation (for total-lengths mean) of conserved regions of backbone sequences in the trimmed orthogroup protein alignment
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test-data/tool-data/plant_tribes/scaffolds/README.txt	Mon Oct 30 09:51:13 2017 -0400
@@ -0,0 +1,3 @@
+For functional tests to work, this directory must contain symlinks to the scaffolds data
+installed into the Galaxy instance to which planemo points via the --galaxy_root parameter.
+This would typically be something like ~/galaxy/tool-data/plant_tribes/scaffolds/22Gv1.1.
--- a/utils.py	Thu Aug 24 13:10:29 2017 -0400
+++ b/utils.py	Mon Oct 30 09:51:13 2017 -0400
@@ -27,7 +27,7 @@
     return fstderr, fherr, fstdout, fhout
 
 
-def move_directory_files(source_dir, destination_dir, copy=False):
+def move_directory_files(source_dir, destination_dir, copy=False, remove_source_dir=False):
     source_directory = os.path.abspath(source_dir)
     destination_directory = os.path.abspath(destination_dir)
     if not os.path.isdir(destination_directory):
@@ -38,6 +38,8 @@
             shutil.copy(source_entry, destination_directory)
         else:
             shutil.move(source_entry, destination_directory)
+    if remove_source_dir:
+        os.rmdir(source_directory)
 
 
 def run_command(cmd):
@@ -52,29 +54,3 @@
 
 def stop_err(msg):
     sys.exit(msg)
-
-
-def write_html_output(output, title, dir):
-    with open(output, 'w') as fh:
-        dir_items = sorted(os.listdir(dir))
-        # Directories can only contain either files or directories,
-        # but not both.
-        if len(dir_items) > 0:
-            item_path = os.path.join(dir, dir_items[0])
-            if os.path.isdir(item_path):
-                header = 'Directories'
-            else:
-                header = 'Datasets'
-        else:
-            header = ''
-        fh.write('<html><head><h3>%s: %d items</h3></head>\n' % (title, len(dir_items)))
-        fh.write('<body><p/><table cellpadding="2">\n')
-        fh.write('<tr><b>%s</th></b>\n' % header)
-        for index, fname in enumerate(dir_items):
-            if index % 2 == 0:
-                bgcolor = '#D8D8D8'
-            else:
-                bgcolor = '#FFFFFF'
-            link = '<a href="%s" type="text/plain">%s</a>\n' % (fname, fname)
-            fh.write('<tr bgcolor="%s"><td>%s</td></tr>\n' % (bgcolor, link))
-        fh.write('</table></body></html>\n')