Mercurial > repos > peterjc > fastq_filter_by_id

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tools/fastq/fastq_filter_by_id.py	Tue Jun 07 17:23:26 2011 -0400
@@ -0,0 +1,93 @@
+#!/usr/bin/env python
+"""Filter a FASTQ file with IDs from a tabular file, e.g. from BLAST.
+
+Takes five command line options, tabular filename, ID column numbers
+(comma separated list using one based counting), input FASTA filename, and
+two output FASTA filenames (for records with and without the given IDs).
+
+If either output filename is just a minus sign, that file is not created.
+This is intended to allow output for just the matched (or just the non-matched)
+records.
+
+Note in the default NCBI BLAST+ tabular output, the query sequence ID is
+in column one, and the ID of the match from the database is in column two.
+Here sensible values for the column numbers would therefore be "1" or "2".
+
+This script is copyright 2010 by Peter Cock, SCRI, UK. All rights reserved.
+See accompanying text file for licence details (MIT/BSD style).
+
+This is version 0.0.2 of the script.
+"""
+import sys
+from galaxy_utils.sequence.fastq import fastqReader, fastqWriter
+
+def stop_err( msg ):
+    sys.stderr.write( msg )
+    sys.exit()
+
+#Parse Command Line
+try:
+    tabular_file, cols_arg, in_file, out_positive_file, out_negative_file = sys.argv[1:]
+except ValueError:
+    stop_err("Expected five arguments, got %i:\n%s" % (len(sys.argv)-1, " ".join(sys.argv)))
+try:
+    columns = [int(arg)-1 for arg in cols_arg.split(",")]
+except ValueError:
+    stop_err("Expected list of columns (comma separated integers), got %s" % cols_arg)
+
+#Read tabular file and record all specified identifiers
+ids = set()
+handle = open(tabular_file, "rU")
+if len(columns)>1:
+    #General case of many columns
+    for line in handle:
+        if line.startswith("#"):
+            #Ignore comments
+            continue
+        parts = line.rstrip("\n").split("\t")
+        for col in columns:
+            ids.add(parts[col])
+    print "Using %i IDs from %i columns of tabular file" % (len(ids), len(columns))
+else:
+    #Single column, special case speed up
+    col = columns[0]
+    for line in handle:
+        if not line.startswith("#"):
+            ids.add(line.rstrip("\n").split("\t")[col])
+    print "Using %i IDs from tabular file" % (len(ids))
+handle.close()
+
+#Write filtered FASTQ file based on IDs from tabular file
+reader = fastqReader(open(in_file, "rU"))
+if out_positive_file != "-" and out_negative_file != "-":
+    print "Generating two FASTQ files"
+    positive_writer = fastqWriter(open(out_positive_file, "w"))
+    negative_writer = fastqWriter(open(out_negative_file, "w"))
+    for record in reader:
+        #The [1:] is because the fastaReader leaves the @ on the identifer.
+        if record.identifier and record.identifier.split()[0][1:] in ids:
+            positive_writer.write(record)
+        else:
+            negative_writer.write(record)
+    positive_writer.close()
+    negative_writer.close()
+elif out_positive_file != "-":
+    print "Generating matching FASTQ file"
+    positive_writer = fastqWriter(open(out_positive_file, "w"))
+    for record in reader:
+        #The [1:] is because the fastaReader leaves the @ on the identifer.
+        if record.identifier and record.identifier.split()[0][1:] in ids:
+            positive_writer.write(record)
+    positive_writer.close()
+elif out_negative_file != "-":
+    print "Generating non-matching FASTQ file"
+    negative_writer = fastqWriter(open(out_negative_file, "w"))
+    for record in reader:
+        #The [1:] is because the fastaReader leaves the @ on the identifer.
+        if not record.identifier or record.identifier.split()[0][1:] not in ids:
+            negative_writer.write(record)
+    positive_writer.close()
+    negative_writer.close()
+else:
+    stop_err("Neither output file requested")
+reader.close()
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tools/fastq/fastq_filter_by_id.txt	Tue Jun 07 17:23:26 2011 -0400
@@ -0,0 +1,77 @@
+Galaxy tool to filter FASTQ sequences by ID
+===========================================
+
+This tool is copyright 2010 by Peter Cock, SCRI, UK. All rights reserved.
+See the licence text below.
+
+This tool is a short Python script (using the Galaxy library functions) which
+divides a FASTQ file in two, those sequences with or without an ID present in
+the specified column(s) of a tabular file. Example uses include filtering based
+on search results from a tool like NCBI BLAST before assembly.
+
+There are just two files to install:
+
+* fastq_filter_by_id.py (the Python script)
+* fastq_filter_by_id.xml (the Galaxy tool definition)
+
+The suggested location is next to the similarly named fastq_filter.py and
+fastq_filter.xml files which are included with Galaxy, i.e. in the Galaxy
+folder tools/fastq
+
+You will also need to modify the tools_conf.xml file to tell Galaxy to offer
+the tool. The suggested location is next to the fastq_filter.xml entry. Simply
+add the line:
+
+<tool file="fastq/fastq_filter_by_id.xml" />
+
+That's it.
+
+
+History
+=======
+
+v0.0.1 - Initial verion (not publicly released)
+v0.0.2 - Allow both, just pos or just neg output files
+       - Preserve the FASTQ variant in the XML wrapper
+
+
+Developers
+==========
+
+This script and similar versions for FASTA and SFF files are currently being
+developed on the following hg branch:
+http://bitbucket.org/peterjc/galaxy-central/src/fasta_filter
+
+For making the "Galaxy Tool Shed" http://community.g2.bx.psu.edu/ tarball use
+the following command from the Galaxy root folder:
+
+tar -czf fastq_filter_by_id.tar.gz tools/fastq/fastq_filter_by_id.*
+
+Check this worked:
+
+$ tar -tzf fastq_filter_by_id.tar.gz
+fastq/fastq_filter_by_id.py
+fastq/fastq_filter_by_id.txt
+fastq/fastq_filter_by_id.xml
+
+
+Licence (MIT/BSD style)
+=======================
+
+Permission to use, copy, modify, and distribute this software and its
+documentation with or without modifications and for any purpose and
+without fee is hereby granted, provided that any copyright notices
+appear in all copies and that both those copyright notices and this
+permission notice appear in supporting documentation, and that the
+names of the contributors or copyright holders not be used in
+advertising or publicity pertaining to distribution of the software
+without specific prior permission.
+
+THE CONTRIBUTORS AND COPYRIGHT HOLDERS OF THIS SOFTWARE DISCLAIM ALL
+WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL THE
+CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY SPECIAL, INDIRECT
+OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE
+OR PERFORMANCE OF THIS SOFTWARE.
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tools/fastq/fastq_filter_by_id.xml	Tue Jun 07 17:23:26 2011 -0400
@@ -0,0 +1,75 @@
+<tool id="fastq_filter_by_id" name="Filter FASTQ by ID" version="0.0.2">
+	<description>from a tabular file</description>
+	<command interpreter="python">
+fastq_filter_by_id.py $input_tabular $columns $input_fastq
+#if $output_choice_cond.output_choice=="both"
+ $output_pos $output_neg
+#elif $output_choice_cond.output_choice=="pos"
+ $output_pos -
+#elif $output_choice_cond.output_choice=="neg"
+ - $output_neg
+#end if
+	</command>
+	<inputs>
+		<param name="input_fastq" type="data" format="fastq" label="FASTQ file to filter on the identifiers"/>
+		<param name="input_tabular" type="data" format="tabular" label="Tabular file containing FASTQ identifiers"/>
+		<param name="columns" type="data_column" data_ref="input_tabular" multiple="True" numerical="False" label="Column(s) containing FASTA identifiers" help="Multi-select list - hold the appropriate key while clicking to select multiple columns">
+			<validator type="no_options" message="Pick at least one column"/>
+		</param>
+		<conditional name="output_choice_cond">
+			<param name="output_choice" type="select" label="Output positive matches, negative matches, or both?">
+				<option value="both">Both positive matches (ID on list) and negative matches (ID not on list), as two FASTA files</option>
+				<option value="pos">Just positive matches (ID on list), as a single FASTA file</option>
+				<option value="neg">Just negative matches (ID not on list), as a single FASTA file</option>
+			</param>
+			<!-- Seems need these dummy entries here, compare this to indels/indel_sam2interval.xml -->
+			<when value="both" />
+			<when value="pos" />
+			<when value="neg" />
+		</conditional>
+	</inputs>
+	<outputs>
+		<data name="output_pos" format="fastq" label="With matched ID">
+            <!-- TODO - Replace this with format="input:input_fastq" if/when that works -->
+            <change_format>
+                <when input_dataset="input_fastq" attribute="extension" value="fastqsanger" format="fastqsanger" />
+                <when input_dataset="input_fastq" attribute="extension" value="fastqsolexa" format="fastqsolexa" />
+                <when input_dataset="input_fastq" attribute="extension" value="fastqillumina" format="fastqillumina" />
+                <when input_dataset="input_fastq" attribute="extension" value="fastqcssanger" format="fastqcssanger" />
+            </change_format>
+			<filter>output_choice_cond["output_choice"] != "neg"</filter>
+		</data>
+		<data name="output_neg" format="fastq" label="Without matched ID">
+            <!-- TODO - Replace this with format="input:input_fastq" if/when that works -->
+            <change_format>
+                <when input_dataset="input_fastq" attribute="extension" value="fastqsanger" format="fastqsanger" />
+                <when input_dataset="input_fastq" attribute="extension" value="fastqsolexa" format="fastqsolexa" />
+                <when input_dataset="input_fastq" attribute="extension" value="fastqillumina" format="fastqillumina" />
+                <when input_dataset="input_fastq" attribute="extension" value="fastqcssanger" format="fastqcssanger" />
+            </change_format>
+			<filter>output_choice_cond["output_choice"] != "pos"</filter>
+		</data>
+	</outputs>
+	<tests>
+	</tests>
+	<help>
+
+**What it does**
+
+By default it divides a FASTQ file in two, those sequences with or without an
+ID present in the tabular file column(s) specified. You can opt to have a
+single output file of just the matching records, or just the non-matching ones.
+
+Note that the order of sequences in the original FASTA file is preserved.
+Also, if any sequences share an identifier, duplicates are not removed.
+
+**Example Usage**
+
+You may have performed some kind of contamination search, for example running
+BLASTN against a database of cloning vectors or bacteria, giving you a tabular
+file containing read identifiers. You could use this tool to extract only the
+reads without BLAST matches (i.e. those which do not match your contaminant
+database).
+
+	</help>
+</tool>