Mercurial > repos > peterjc > seq_filter_by_id
changeset 3:44ab4c0f7683 draft
Uploaded v0.0.6, automatic dependency on Biopython 1.62, new README file, citation information, MIT licence
| author | peterjc |
|---|---|
| date | Fri, 11 Oct 2013 04:37:12 -0400 |
| parents | abdd608c869b |
| children | 1c36cf8ef133 |
| files | tools/filters/seq_filter_by_id.py tools/filters/seq_filter_by_id.txt tools/filters/seq_filter_by_id.xml tools/seq_filter_by_id/README.rst tools/seq_filter_by_id/repository_dependencies.xml tools/seq_filter_by_id/seq_filter_by_id.py tools/seq_filter_by_id/seq_filter_by_id.xml |
| diffstat | 7 files changed, 515 insertions(+), 435 deletions(-) [+] |
line wrap: on
line diff
--- a/tools/filters/seq_filter_by_id.py Wed Apr 24 11:34:12 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,233 +0,0 @@ -#!/usr/bin/env python -"""Filter a FASTA, FASTQ or SSF file with IDs from a tabular file. - -Takes six command line options, tabular filename, ID column numbers (comma -separated list using one based counting), input filename, input type (e.g. -FASTA or SFF) and two output filenames (for records with and without the -given IDs, same format as input sequence file). - -If either output filename is just a minus sign, that file is not created. -This is intended to allow output for just the matched (or just the non-matched) -records. - -When filtering an SFF file, any Roche XML manifest in the input file is -preserved in both output files. - -Note in the default NCBI BLAST+ tabular output, the query sequence ID is -in column one, and the ID of the match from the database is in column two. -Here sensible values for the column numbers would therefore be "1" or "2". - -This tool is a short Python script which requires Biopython 1.54 or later -for SFF file support. If you use this tool in scientific work leading to a -publication, please cite the Biopython application note: - -Cock et al 2009. Biopython: freely available Python tools for computational -molecular biology and bioinformatics. Bioinformatics 25(11) 1422-3. -http://dx.doi.org/10.1093/bioinformatics/btp163 pmid:19304878. - -This script is copyright 2010-2013 by Peter Cock, The James Hutton Institute -(formerly the Scottish Crop Research Institute, SCRI), UK. All rights reserved. -See accompanying text file for licence details (MIT/BSD style). - -This is version 0.0.5 of the script, use -v or --version to get the version. -""" -import sys - -def stop_err(msg, err=1): - sys.stderr.write(msg.rstrip() + "\n") - sys.exit(err) - -if "-v" in sys.argv or "--version" in sys.argv: - print "v0.0.5" - sys.exit(0) - -#Parse Command Line -try: - tabular_file, cols_arg, in_file, seq_format, out_positive_file, out_negative_file = sys.argv[1:] -except ValueError: - stop_err("Expected six arguments (tab file, columns, in seq, seq format, out pos, out neg), got %i:\n%s" % (len(sys.argv)-1, " ".join(sys.argv))) -try: - columns = [int(arg)-1 for arg in cols_arg.split(",")] -except ValueError: - stop_err("Expected list of columns (comma separated integers), got %s" % cols_arg) -if min(columns) < 0: - stop_err("Expect one-based column numbers (not zero-based counting), got %s" % cols_arg) - -if out_positive_file == "-" and out_negative_file == "-": - stop_err("Neither output file requested") - - -#Read tabular file and record all specified identifiers -ids = set() -handle = open(tabular_file, "rU") -if len(columns)>1: - #General case of many columns - for line in handle: - if line.startswith("#"): - #Ignore comments - continue - parts = line.rstrip("\n").split("\t") - for col in columns: - ids.add(parts[col]) - print "Using %i IDs from %i columns of tabular file" % (len(ids), len(columns)) -else: - #Single column, special case speed up - col = columns[0] - for line in handle: - if not line.startswith("#"): - ids.add(line.rstrip("\n").split("\t")[col]) - print "Using %i IDs from tabular file" % (len(ids)) -handle.close() - - -def crude_fasta_iterator(handle): - """Yields tuples, record ID and the full record as a string.""" - while True: - line = handle.readline() - if line == "": - return # Premature end of file, or just empty? - if line[0] == ">": - break - - no_id_warned = False - while True: - if line[0] != ">": - raise ValueError( - "Records in Fasta files should start with '>' character") - try: - id = line[1:].split(None, 1)[0] - except IndexError: - if not no_id_warned: - sys.stderr.write("WARNING - Malformed FASTA entry with no identifier\n") - no_id_warned = True - id = None - lines = [line] - line = handle.readline() - while True: - if not line: - break - if line[0] == ">": - break - lines.append(line) - line = handle.readline() - yield id, "".join(lines) - if not line: - return # StopIteration - - -def fasta_filter(in_file, pos_file, neg_file, wanted): - """FASTA filter producing 60 character line wrapped outout.""" - pos_count = neg_count = 0 - #Galaxy now requires Python 2.5+ so can use with statements, - with open(in_file) as in_handle: - #Doing the if statement outside the loop for speed - #(with the downside of three very similar loops). - if pos_file != "-" and neg_file != "-": - print "Generating two FASTA files" - with open(pos_file, "w") as pos_handle: - with open(neg_file, "w") as neg_handle: - for identifier, record in crude_fasta_iterator(in_handle): - if identifier in wanted: - pos_handle.write(record) - pos_count += 1 - else: - neg_handle.write(record) - neg_count += 1 - elif pos_file != "-": - print "Generating matching FASTA file" - with open(pos_file, "w") as pos_handle: - for identifier, record in crude_fasta_iterator(in_handle): - if identifier in wanted: - pos_handle.write(record) - pos_count += 1 - else: - neg_count += 1 - else: - print "Generating non-matching FASTA file" - assert neg_file != "-" - with open(neg_file, "w") as neg_handle: - for identifier, record in crude_fasta_iterator(in_handle): - if identifier in wanted: - pos_count += 1 - else: - neg_handle.write(record) - neg_count += 1 - return pos_count, neg_count - - -if seq_format.lower()=="sff": - #Now write filtered SFF file based on IDs from BLAST file - try: - from Bio.SeqIO.SffIO import SffIterator, SffWriter - except ImportError: - stop_err("SFF filtering requires Biopython 1.54 or later") - - try: - from Bio.SeqIO.SffIO import ReadRocheXmlManifest - except ImportError: - #Prior to Biopython 1.56 this was a private function - from Bio.SeqIO.SffIO import _sff_read_roche_index_xml as ReadRocheXmlManifest - in_handle = open(in_file, "rb") #must be binary mode! - try: - manifest = ReadRocheXmlManifest(in_handle) - except ValueError: - manifest = None - #This makes two passes though the SFF file with isn't so efficient, - #but this makes the code simple. - pos_count = neg_count = 0 - if out_positive_file != "-": - out_handle = open(out_positive_file, "wb") - writer = SffWriter(out_handle, xml=manifest) - in_handle.seek(0) #start again after getting manifest - pos_count = writer.write_file(rec for rec in SffIterator(in_handle) if rec.id in ids) - out_handle.close() - if out_negative_file != "-": - out_handle = open(out_negative_file, "wb") - writer = SffWriter(out_handle, xml=manifest) - in_handle.seek(0) #start again - neg_count = writer.write_file(rec for rec in SffIterator(in_handle) if rec.id not in ids) - out_handle.close() - #And we're done - in_handle.close() - #At the time of writing, Galaxy doesn't show SFF file read counts, - #so it is useful to put them in stdout and thus shown in job info. - print "%i with and %i without specified IDs" % (pos_count, neg_count) -elif seq_format.lower()=="fasta": - #Write filtered FASTA file based on IDs from tabular file - pos_count, neg_count = fasta_filter(in_file, out_positive_file, out_negative_file, ids) - print "%i with and %i without specified IDs" % (pos_count, neg_count) -elif seq_format.lower().startswith("fastq"): - #Write filtered FASTQ file based on IDs from tabular file - from galaxy_utils.sequence.fastq import fastqReader, fastqWriter - reader = fastqReader(open(in_file, "rU")) - if out_positive_file != "-" and out_negative_file != "-": - print "Generating two FASTQ files" - positive_writer = fastqWriter(open(out_positive_file, "w")) - negative_writer = fastqWriter(open(out_negative_file, "w")) - for record in reader: - #The [1:] is because the fastaReader leaves the > on the identifier. - if record.identifier and record.identifier.split()[0][1:] in ids: - positive_writer.write(record) - else: - negative_writer.write(record) - positive_writer.close() - negative_writer.close() - elif out_positive_file != "-": - print "Generating matching FASTQ file" - positive_writer = fastqWriter(open(out_positive_file, "w")) - for record in reader: - #The [1:] is because the fastaReader leaves the > on the identifier. - if record.identifier and record.identifier.split()[0][1:] in ids: - positive_writer.write(record) - positive_writer.close() - elif out_negative_file != "-": - print "Generating non-matching FASTQ file" - negative_writer = fastqWriter(open(out_negative_file, "w")) - for record in reader: - #The [1:] is because the fastaReader leaves the > on the identifier. - if not record.identifier or record.identifier.split()[0][1:] not in ids: - negative_writer.write(record) - negative_writer.close() - reader.close() -else: - stop_err("Unsupported file type %r" % seq_format)
--- a/tools/filters/seq_filter_by_id.txt Wed Apr 24 11:34:12 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,89 +0,0 @@ -Galaxy tool to filter FASTA, FASTQ or SFF sequences by ID -========================================================= - -This tool is copyright 2010-2013 by Peter Cock, The James Hutton Institute -(formerly SCRI, Scottish Crop Research Institute), UK. All rights reserved. -See the licence text below. - -This tool is a short Python script (using both the Galaxy and Biopython library -functions) which divides a FASTA, FASTQ, or SFF file in two, those sequences with -or without an ID present in the specified column(s) of a tabular file. Example uses -include filtering based on search results from a tool like NCBI BLAST before -assembly. - - -Installation -============ - -There are just two files to install: - -* seq_filter_by_id.py (the Python script) -* seq_filter_by_id.xml (the Galaxy tool definition) - -The suggested location is in the Galaxy folder tools/filters next to the tool -for calling sff_extract.py for converting SFF to FASTQ or FASTA + QUAL. - -You will also need to modify the tools_conf.xml file to tell Galaxy to offer the -tool. One suggested location is in the filters section. Simply add the line: - -<tool file="filters/sff_filter_by_id.xml" /> - -You will also need to install Biopython 1.54 or later. That's it. - - -History -======= - -v0.0.1 - Initial version, combining three separate scripts for each file format. -v0.0.4 - Record script version when run from Galaxy. - - Faster FASTA code which preserves the original line wrapping. - - Basic unit test included. -v0.0.5 - Check for errors using Python script's return code. - - Cope with malformed FASTA entries without an identifier. - - -Developers -========== - -This script and related tools are being developed on the following hg branch: -http://bitbucket.org/peterjc/galaxy-central/src/tools - -This incorporates the previously used hg branch: -http://bitbucket.org/peterjc/galaxy-central/src/fasta_filter - -For making the "Galaxy Tool Shed" http://toolshed.g2.bx.psu.edu/ tarball use -the following command from the Galaxy root folder: - -$ tar -czf seq_filter_by_id.tar.gz tools/filters/seq_filter_by_id.* test-data/k12_ten_proteins.fasta test-data/k12_hypothetical.fasta test-data/k12_hypothetical.tabular - -Check this worked: - -$ tar -tzf seq_filter_by_id.tar.gz -filter/seq_filter_by_id.py -filter/seq_filter_by_id.txt -filter/seq_filter_by_id.xml -test-data/k12_ten_proteins.fasta -test-data/k12_hypothetical.fasta -test-data/k12_hypothetical.tabular - - -Licence (MIT/BSD style) -======================= - -Permission to use, copy, modify, and distribute this software and its -documentation with or without modifications and for any purpose and -without fee is hereby granted, provided that any copyright notices -appear in all copies and that both those copyright notices and this -permission notice appear in supporting documentation, and that the -names of the contributors or copyright holders not be used in -advertising or publicity pertaining to distribution of the software -without specific prior permission. - -THE CONTRIBUTORS AND COPYRIGHT HOLDERS OF THIS SOFTWARE DISCLAIM ALL -WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL THE -CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY SPECIAL, INDIRECT -OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS -OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE -OR PERFORMANCE OF THIS SOFTWARE.
--- a/tools/filters/seq_filter_by_id.xml Wed Apr 24 11:34:12 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,113 +0,0 @@ -<tool id="seq_filter_by_id" name="Filter sequences by ID" version="0.0.5"> - <description>from a tabular file</description> - <version_command interpreter="python">seq_filter_by_id.py --version</version_command> - <command interpreter="python"> -seq_filter_by_id.py $input_tabular $columns $input_file $input_file.ext -#if $output_choice_cond.output_choice=="both" - $output_pos $output_neg -#elif $output_choice_cond.output_choice=="pos" - $output_pos - -#elif $output_choice_cond.output_choice=="neg" - - $output_neg -#end if - </command> - <stdio> - <!-- Anything other than zero is an error --> - <exit_code range="1:" /> - <exit_code range=":-1" /> - </stdio> - <inputs> - <param name="input_file" type="data" format="fasta,fastq,sff" label="Sequence file to filter on the identifiers" help="FASTA, FASTQ, or SFF format." /> - <param name="input_tabular" type="data" format="tabular" label="Tabular file containing sequence identifiers"/> - <param name="columns" type="data_column" data_ref="input_tabular" multiple="True" numerical="False" label="Column(s) containing sequence identifiers" help="Multi-select list - hold the appropriate key while clicking to select multiple columns"> - <validator type="no_options" message="Pick at least one column"/> - </param> - <conditional name="output_choice_cond"> - <param name="output_choice" type="select" label="Output positive matches, negative matches, or both?"> - <option value="both">Both positive matches (ID on list) and negative matches (ID not on list), as two files</option> - <option value="pos">Just positive matches (ID on list), as a single file</option> - <option value="neg">Just negative matches (ID not on list), as a single file</option> - </param> - <!-- Seems need these dummy entries here, compare this to indels/indel_sam2interval.xml --> - <when value="both" /> - <when value="pos" /> - <when value="neg" /> - </conditional> - </inputs> - <outputs> - <data name="output_pos" format="fasta" label="With matched ID"> - <!-- TODO - Replace this with format="input:input_fastq" if/when that works --> - <change_format> - <when input_dataset="input_file" attribute="extension" value="sff" format="sff" /> - <when input_dataset="input_file" attribute="extension" value="fastq" format="fastq" /> - <when input_dataset="input_file" attribute="extension" value="fastqsanger" format="fastqsanger" /> - <when input_dataset="input_file" attribute="extension" value="fastqsolexa" format="fastqsolexa" /> - <when input_dataset="input_file" attribute="extension" value="fastqillumina" format="fastqillumina" /> - <when input_dataset="input_file" attribute="extension" value="fastqcssanger" format="fastqcssanger" /> - </change_format> - <filter>output_choice_cond["output_choice"] != "neg"</filter> - </data> - <data name="output_neg" format="fasta" label="Without matched ID"> - <!-- TODO - Replace this with format="input:input_fastq" if/when that works --> - <change_format> - <when input_dataset="input_file" attribute="extension" value="sff" format="sff" /> - <when input_dataset="input_file" attribute="extension" value="fastq" format="fastq" /> - <when input_dataset="input_file" attribute="extension" value="fastqsanger" format="fastqsanger" /> - <when input_dataset="input_file" attribute="extension" value="fastqsolexa" format="fastqsolexa" /> - <when input_dataset="input_file" attribute="extension" value="fastqillumina" format="fastqillumina" /> - <when input_dataset="input_file" attribute="extension" value="fastqcssanger" format="fastqcssanger" /> - </change_format> - <filter>output_choice_cond["output_choice"] != "pos"</filter> - </data> - </outputs> - <tests> - <test> - <param name="input_file" value="k12_ten_proteins.fasta" ftype="fasta" /> - <param name="input_tabular" value="k12_hypothetical.tabular" ftype="tabular" /> - <param name="columns" value="1" /> - <param name="output_choice" value="pos" /> - <output name="output_pos" file="k12_hypothetical.fasta" ftype="fasta" /> - </test> - </tests> - <requirements> - <requirement type="python-module">Bio</requirement> - </requirements> - <help> - -**What it does** - -By default it divides a FASTA, FASTQ or Standard Flowgram Format (SFF) file in -two, those sequences with or without an ID present in the tabular file column(s) -specified. You can opt to have a single output file of just the matching records, -or just the non-matching ones. - -Note that the order of sequences in the original sequence file is preserved, as -is any Roche XML Manifest in an SFF file. Also, if any sequences share an -identifier (which would be very unusual in SFF files), duplicates are not removed. - -**Example Usage** - -You may have performed some kind of contamination search, for example running -BLASTN against a database of cloning vectors or bacteria, giving you a tabular -file containing read identifiers. You could use this tool to extract only the -reads without BLAST matches (i.e. those which do not match your contaminant -database). - -You may have a file of FASTA sequences which has been used with some analysis -tool giving tabular output, which has then been filtered on some criteria. -You can then use this tool to divide the original FASTA file into those entries -matching or not matching your criteria (those with or without their identifier -in the filtered tabular file). - -**Citation** - -This tool uses Biopython to read and write SFF files. If you use this tool in -scientific work leading to a publication, please cite the Biopython application -note (and Galaxy too of course): - -Cock et al 2009. Biopython: freely available Python tools for computational -molecular biology and bioinformatics. Bioinformatics 25(11) 1422-3. -http://dx.doi.org/10.1093/bioinformatics/btp163 pmid:19304878. - - </help> -</tool>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tools/seq_filter_by_id/README.rst Fri Oct 11 04:37:12 2013 -0400 @@ -0,0 +1,124 @@ +Galaxy tool to filter FASTA, FASTQ or SFF sequences by ID +========================================================= + +This tool is copyright 2010-2013 by Peter Cock, The James Hutton Institute +(formerly SCRI, Scottish Crop Research Institute), UK. All rights reserved. +See the licence text below. + +This tool is a short Python script (using both the Galaxy and Biopython library +functions) which divides a FASTA, FASTQ, or SFF file in two, those sequences with +or without an ID present in the specified column(s) of a tabular file. Example uses +include filtering based on search results from a tool like NCBI BLAST before +assembly. + +This tool is available from the Galaxy Tool Shed at: + +* http://toolshed.g2.bx.psu.edu/view/peterjc/seq_filter_by_id + +See also sister tools: + +* http://toolshed.g2.bx.psu.edu/view/peterjc/seq_select_by_id +* http://toolshed.g2.bx.psu.edu/view/peterjc/seq_rename + + +Automated Installation +====================== + +This should be straightforward using the Galaxy Tool Shed, which should be +able to automatically install the dependency on Biopython, and then install +this tool and run its unit tests. + + +Manual Installation +=================== + +There are just two files to install to use this tool from within Galaxy: + +* seq_filter_by_id.py (the Python script) +* seq_filter_by_id.xml (the Galaxy tool definition) + +The suggested location is a dedicated tools/seq_filter_by_id folder. + +You will also need to modify the tools_conf.xml file to tell Galaxy to offer the +tool. One suggested location is in the filters section. Simply add the line:: + + <tool file="seq_filter_by_id/seq_filter_by_id.xml" /> + +If you wish to run the unit tests, also add this to tools_conf.xml.sample +and move/copy the test-data files under Galaxy's test-data folder. Then:: + + $ ./run_functional_tests.sh -id seq_filter_by_id + +You will also need to install Biopython 1.54 or later. That's it. + + +History +======= + +======= ====================================================================== +Version Changes +------- ---------------------------------------------------------------------- +v0.0.1 - Initial version, combining three separate scripts for each file format. +v0.0.4 - Record script version when run from Galaxy. + - Faster FASTA code which preserves the original line wrapping. + - Basic unit test included. +v0.0.5 - Check for errors using Python script's return code. + - Cope with malformed FASTA entries without an identifier. +v0.0.6 - Link to Tool Shed added to help text and this documentation. + - Automated installation of the Biopython dependency. + - Use reStructuredText for this README file. + - Adopt standard MIT License. + - Updated citation information (Cock et al. 2013). + - Development moved to GitHub, https://github.com/peterjc/pico_galaxy + - Renamed folder and adopted README.rst naming. +======= ====================================================================== + + + +Developers +========== + +This script and related tools were initially developed on the following hg branches: +http://bitbucket.org/peterjc/galaxy-central/src/fasta_filter +http://bitbucket.org/peterjc/galaxy-central/src/tools + +Development has now moved to a dedicated GitHub repository: +https://github.com/peterjc/pico_galaxy/tree/master/tools + +For making the "Galaxy Tool Shed" http://toolshed.g2.bx.psu.edu/ tarball use +the following command from the Galaxy root folder:: + + $ tar -czf seq_filter_by_id.tar.gz tools/seq_filter_by_id/README.rst tools/seq_filter_by_id/seq_filter_by_id.* tools/seq_filter_by_id/repository_dependencies.xml test-data/k12_ten_proteins.fasta test-data/k12_hypothetical.fasta test-data/k12_hypothetical.tabular + +Check this worked:: + + $ tar -tzf seq_filter_by_id.tar.gz + tools/seq_filter_by_id/README.rst + tools/seq_filter_by_id/seq_filter_by_id.py + tools/seq_filter_by_id/seq_filter_by_id.xml + tools/seq_filter_by_id/repository_dependencies.xml + test-data/k12_ten_proteins.fasta + test-data/k12_hypothetical.fasta + test-data/k12_hypothetical.tabular + + +Licence (MIT) +============= + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE.
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tools/seq_filter_by_id/repository_dependencies.xml Fri Oct 11 04:37:12 2013 -0400 @@ -0,0 +1,6 @@ +<?xml version="1.0"?> +<repositories description="This requires Biopython as a dependency."> +<!-- Leave out the tool shed and revision to get the current + tool shed and latest revision at the time of upload --> +<repository changeset_revision="3e82cbc44886" name="package_biopython_1_62" owner="biopython" toolshed="http://toolshed.g2.bx.psu.edu" /> +</repositories>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tools/seq_filter_by_id/seq_filter_by_id.py Fri Oct 11 04:37:12 2013 -0400 @@ -0,0 +1,260 @@ +#!/usr/bin/env python +"""Filter a FASTA, FASTQ or SSF file with IDs from a tabular file. + +Takes six command line options, tabular filename, ID column numbers (comma +separated list using one based counting), input filename, input type (e.g. +FASTA or SFF) and two output filenames (for records with and without the +given IDs, same format as input sequence file). + +If either output filename is just a minus sign, that file is not created. +This is intended to allow output for just the matched (or just the non-matched) +records. + +When filtering an SFF file, any Roche XML manifest in the input file is +preserved in both output files. + +Note in the default NCBI BLAST+ tabular output, the query sequence ID is +in column one, and the ID of the match from the database is in column two. +Here sensible values for the column numbers would therefore be "1" or "2". + +This tool is a short Python script which requires Biopython 1.54 or later +for SFF file support. If you use this tool in scientific work leading to a +publication, please cite the Biopython application note: + +Cock et al 2009. Biopython: freely available Python tools for computational +molecular biology and bioinformatics. Bioinformatics 25(11) 1422-3. +http://dx.doi.org/10.1093/bioinformatics/btp163 pmid:19304878. + +This script is copyright 2010-2013 by Peter Cock, The James Hutton Institute +(formerly the Scottish Crop Research Institute, SCRI), UK. All rights reserved. +See accompanying text file for licence details (MIT license). + +This is version 0.1.0 of the script, use -v or --version to get the version. +""" +import os +import sys + +def stop_err(msg, err=1): + sys.stderr.write(msg.rstrip() + "\n") + sys.exit(err) + +if "-v" in sys.argv or "--version" in sys.argv: + print "v0.1.0" + sys.exit(0) + +#Parse Command Line +if len(sys.argv) - 1 < 7 or len(sys.argv) % 2 == 1: + stop_err("Expected 7 or more arguments, 5 required " + "(in seq, seq format, out pos, out neg, logic) " + "then one or more pairs (tab file, columns), " + "got %i:\n%s" % (len(sys.argv)-1, " ".join(sys.argv))) + +in_file, seq_format, out_positive_file, out_negative_file, logic = sys.argv[1:6] + +if not os.path.isfile(in_file): + stop_err("Missing input file %r" % in_file) +if out_positive_file == "-" and out_negative_file == "-": + stop_err("Neither output file requested") +if logic not in ["UNION", "INTERSECTION"]: + stop_err("Fifth agrument should be 'UNION' or 'INTERSECTION', not %r" % logic) + +identifiers = [] +for i in range((len(sys.argv) - 6) // 2): + tabular_file = sys.argv[6+2*i] + cols_arg = sys.argv[7+2*i] + if not os.path.isfile(tabular_file): + stop_err("Missing tabular identifier file %r" % tabular_file) + try: + columns = [int(arg)-1 for arg in cols_arg.split(",")] + except ValueError: + stop_err("Expected list of columns (comma separated integers), got %r" % cols_arg) + if min(columns) < 0: + stop_err("Expect one-based column numbers (not zero-based counting), got %r" % cols_arg) + identifiers.append((tabular_file, columns)) + +#Read tabular file(s) and record all specified identifiers +ids = None #Will be a set +for tabular_file, columns in identifiers: + file_ids = set() + handle = open(tabular_file, "rU") + if len(columns)>1: + #General case of many columns + for line in handle: + if line.startswith("#"): + #Ignore comments + continue + parts = line.rstrip("\n").split("\t") + for col in columns: + file_ids.add(parts[col]) + else: + #Single column, special case speed up + col = columns[0] + for line in handle: + if not line.startswith("#"): + file_ids.add(line.rstrip("\n").split("\t")[col]) + print "Using %i IDs from column %s in tabular file" % (len(file_ids), ", ".join(str(col+1) for col in columns)) + if ids is None: + ids = file_ids + if logic == "UNION": + ids.update(file_ids) + else: + ids.intersection_update(file_ids) + handle.close() +if len(identifiers) > 1: + if logic == "UNION": + print "Have %i IDs combined from %i tabular files" % (len(ids), len(identifiers)) + else: + print "Have %i IDs in common from %i tabular files" % (len(ids), len(identifiers)) + + +def crude_fasta_iterator(handle): + """Yields tuples, record ID and the full record as a string.""" + while True: + line = handle.readline() + if line == "": + return # Premature end of file, or just empty? + if line[0] == ">": + break + + no_id_warned = False + while True: + if line[0] != ">": + raise ValueError( + "Records in Fasta files should start with '>' character") + try: + id = line[1:].split(None, 1)[0] + except IndexError: + if not no_id_warned: + sys.stderr.write("WARNING - Malformed FASTA entry with no identifier\n") + no_id_warned = True + id = None + lines = [line] + line = handle.readline() + while True: + if not line: + break + if line[0] == ">": + break + lines.append(line) + line = handle.readline() + yield id, "".join(lines) + if not line: + return # StopIteration + + +def fasta_filter(in_file, pos_file, neg_file, wanted): + """FASTA filter producing 60 character line wrapped outout.""" + pos_count = neg_count = 0 + #Galaxy now requires Python 2.5+ so can use with statements, + with open(in_file) as in_handle: + #Doing the if statement outside the loop for speed + #(with the downside of three very similar loops). + if pos_file != "-" and neg_file != "-": + print "Generating two FASTA files" + with open(pos_file, "w") as pos_handle: + with open(neg_file, "w") as neg_handle: + for identifier, record in crude_fasta_iterator(in_handle): + if identifier in wanted: + pos_handle.write(record) + pos_count += 1 + else: + neg_handle.write(record) + neg_count += 1 + elif pos_file != "-": + print "Generating matching FASTA file" + with open(pos_file, "w") as pos_handle: + for identifier, record in crude_fasta_iterator(in_handle): + if identifier in wanted: + pos_handle.write(record) + pos_count += 1 + else: + neg_count += 1 + else: + print "Generating non-matching FASTA file" + assert neg_file != "-" + with open(neg_file, "w") as neg_handle: + for identifier, record in crude_fasta_iterator(in_handle): + if identifier in wanted: + pos_count += 1 + else: + neg_handle.write(record) + neg_count += 1 + return pos_count, neg_count + + +if seq_format.lower()=="sff": + #Now write filtered SFF file based on IDs from BLAST file + try: + from Bio.SeqIO.SffIO import SffIterator, SffWriter + except ImportError: + stop_err("SFF filtering requires Biopython 1.54 or later") + + try: + from Bio.SeqIO.SffIO import ReadRocheXmlManifest + except ImportError: + #Prior to Biopython 1.56 this was a private function + from Bio.SeqIO.SffIO import _sff_read_roche_index_xml as ReadRocheXmlManifest + in_handle = open(in_file, "rb") #must be binary mode! + try: + manifest = ReadRocheXmlManifest(in_handle) + except ValueError: + manifest = None + #This makes two passes though the SFF file with isn't so efficient, + #but this makes the code simple. + pos_count = neg_count = 0 + if out_positive_file != "-": + out_handle = open(out_positive_file, "wb") + writer = SffWriter(out_handle, xml=manifest) + in_handle.seek(0) #start again after getting manifest + pos_count = writer.write_file(rec for rec in SffIterator(in_handle) if rec.id in ids) + out_handle.close() + if out_negative_file != "-": + out_handle = open(out_negative_file, "wb") + writer = SffWriter(out_handle, xml=manifest) + in_handle.seek(0) #start again + neg_count = writer.write_file(rec for rec in SffIterator(in_handle) if rec.id not in ids) + out_handle.close() + #And we're done + in_handle.close() + #At the time of writing, Galaxy doesn't show SFF file read counts, + #so it is useful to put them in stdout and thus shown in job info. + print "%i with and %i without specified IDs" % (pos_count, neg_count) +elif seq_format.lower()=="fasta": + #Write filtered FASTA file based on IDs from tabular file + pos_count, neg_count = fasta_filter(in_file, out_positive_file, out_negative_file, ids) + print "%i with and %i without specified IDs" % (pos_count, neg_count) +elif seq_format.lower().startswith("fastq"): + #Write filtered FASTQ file based on IDs from tabular file + from galaxy_utils.sequence.fastq import fastqReader, fastqWriter + reader = fastqReader(open(in_file, "rU")) + if out_positive_file != "-" and out_negative_file != "-": + print "Generating two FASTQ files" + positive_writer = fastqWriter(open(out_positive_file, "w")) + negative_writer = fastqWriter(open(out_negative_file, "w")) + for record in reader: + #The [1:] is because the fastaReader leaves the > on the identifier. + if record.identifier and record.identifier.split()[0][1:] in ids: + positive_writer.write(record) + else: + negative_writer.write(record) + positive_writer.close() + negative_writer.close() + elif out_positive_file != "-": + print "Generating matching FASTQ file" + positive_writer = fastqWriter(open(out_positive_file, "w")) + for record in reader: + #The [1:] is because the fastaReader leaves the > on the identifier. + if record.identifier and record.identifier.split()[0][1:] in ids: + positive_writer.write(record) + positive_writer.close() + elif out_negative_file != "-": + print "Generating non-matching FASTQ file" + negative_writer = fastqWriter(open(out_negative_file, "w")) + for record in reader: + #The [1:] is because the fastaReader leaves the > on the identifier. + if not record.identifier or record.identifier.split()[0][1:] not in ids: + negative_writer.write(record) + negative_writer.close() + reader.close() +else: + stop_err("Unsupported file type %r" % seq_format)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tools/seq_filter_by_id/seq_filter_by_id.xml Fri Oct 11 04:37:12 2013 -0400 @@ -0,0 +1,125 @@ +<tool id="seq_filter_by_id" name="Filter sequences by ID" version="0.0.6"> + <description>from a tabular file</description> + <requirements> + <requirement type="package" version="1.62">biopython</requirement> + <requirement type="python-module">Bio</requirement> + </requirements> + <version_command interpreter="python">seq_filter_by_id.py --version</version_command> + <command interpreter="python"> +seq_filter_by_id.py "$input_file" "$input_file.ext" +#if $output_choice_cond.output_choice=="both" + $output_pos $output_neg +#elif $output_choice_cond.output_choice=="pos" + $output_pos - +#elif $output_choice_cond.output_choice=="neg" + - $output_neg +#end if +## TODO - Decide on best way to expose multiple ID files via the XML wrapper. +## Single tabular file, can call the Python script with either UNION or INTERSECTION +UNION "$input_tabular" "$columns" + </command> + <stdio> + <!-- Anything other than zero is an error --> + <exit_code range="1:" /> + <exit_code range=":-1" /> + </stdio> + <inputs> + <param name="input_file" type="data" format="fasta,fastq,sff" label="Sequence file to filter on the identifiers" help="FASTA, FASTQ, or SFF format." /> + <param name="input_tabular" type="data" format="tabular" label="Tabular file containing sequence identifiers"/> + <param name="columns" type="data_column" data_ref="input_tabular" multiple="True" numerical="False" label="Column(s) containing sequence identifiers" help="Multi-select list - hold the appropriate key while clicking to select multiple columns"> + <validator type="no_options" message="Pick at least one column"/> + </param> + <conditional name="output_choice_cond"> + <param name="output_choice" type="select" label="Output positive matches, negative matches, or both?"> + <option value="both">Both positive matches (ID on list) and negative matches (ID not on list), as two files</option> + <option value="pos">Just positive matches (ID on list), as a single file</option> + <option value="neg">Just negative matches (ID not on list), as a single file</option> + </param> + <!-- Seems need these dummy entries here, compare this to indels/indel_sam2interval.xml --> + <when value="both" /> + <when value="pos" /> + <when value="neg" /> + </conditional> + </inputs> + <outputs> + <data name="output_pos" format="fasta" label="With matched ID"> + <!-- TODO - Replace this with format="input:input_fastq" if/when that works --> + <change_format> + <when input_dataset="input_file" attribute="extension" value="sff" format="sff" /> + <when input_dataset="input_file" attribute="extension" value="fastq" format="fastq" /> + <when input_dataset="input_file" attribute="extension" value="fastqsanger" format="fastqsanger" /> + <when input_dataset="input_file" attribute="extension" value="fastqsolexa" format="fastqsolexa" /> + <when input_dataset="input_file" attribute="extension" value="fastqillumina" format="fastqillumina" /> + <when input_dataset="input_file" attribute="extension" value="fastqcssanger" format="fastqcssanger" /> + </change_format> + <filter>output_choice_cond["output_choice"] != "neg"</filter> + </data> + <data name="output_neg" format="fasta" label="Without matched ID"> + <!-- TODO - Replace this with format="input:input_fastq" if/when that works --> + <change_format> + <when input_dataset="input_file" attribute="extension" value="sff" format="sff" /> + <when input_dataset="input_file" attribute="extension" value="fastq" format="fastq" /> + <when input_dataset="input_file" attribute="extension" value="fastqsanger" format="fastqsanger" /> + <when input_dataset="input_file" attribute="extension" value="fastqsolexa" format="fastqsolexa" /> + <when input_dataset="input_file" attribute="extension" value="fastqillumina" format="fastqillumina" /> + <when input_dataset="input_file" attribute="extension" value="fastqcssanger" format="fastqcssanger" /> + </change_format> + <filter>output_choice_cond["output_choice"] != "pos"</filter> + </data> + </outputs> + <tests> + <test> + <param name="input_file" value="k12_ten_proteins.fasta" ftype="fasta" /> + <param name="input_tabular" value="k12_hypothetical.tabular" ftype="tabular" /> + <param name="columns" value="1" /> + <param name="output_choice" value="pos" /> + <output name="output_pos" file="k12_hypothetical.fasta" ftype="fasta" /> + </test> + </tests> + <help> +**What it does** + +By default it divides a FASTA, FASTQ or Standard Flowgram Format (SFF) file in +two, those sequences with or without an ID present in the tabular file column(s) +specified. You can opt to have a single output file of just the matching records, +or just the non-matching ones. + +Note that the order of sequences in the original sequence file is preserved, as +is any Roche XML Manifest in an SFF file. Also, if any sequences share an +identifier (which would be very unusual in SFF files), duplicates are not removed. + +**Example Usage** + +You may have performed some kind of contamination search, for example running +BLASTN against a database of cloning vectors or bacteria, giving you a tabular +file containing read identifiers. You could use this tool to extract only the +reads without BLAST matches (i.e. those which do not match your contaminant +database). + +You may have a file of FASTA sequences which has been used with some analysis +tool giving tabular output, which has then been filtered on some criteria. +You can then use this tool to divide the original FASTA file into those entries +matching or not matching your criteria (those with or without their identifier +in the filtered tabular file). + +**References** + +If you use this Galaxy tool in work leading to a scientific publication please +cite the following papers: + +Peter J.A. Cock, Björn A. Grüning, Konrad Paszkiewicz and Leighton Pritchard (2013). +Galaxy tools and workflows for sequence analysis with applications +in molecular plant pathology. PeerJ 1:e167 +http://dx.doi.org/10.7717/peerj.167 + +This tool uses Biopython to read and write SFF files, so you may also wish to +cite the Biopython application note (and Galaxy too of course): + +Cock et al (2009). Biopython: freely available Python tools for computational +molecular biology and bioinformatics. Bioinformatics 25(11) 1422-3. +http://dx.doi.org/10.1093/bioinformatics/btp163 pmid:19304878. + +This tool is available to install into other Galaxy Instances via the Galaxy +Tool Shed at http://toolshed.g2.bx.psu.edu/view/peterjc/seq_filter_by_id + </help> +</tool>