# HG changeset patch # User peterjc # Date 1382952105 14400 # Node ID 6842c0c7bc70b6d2421515fe4f16a4008d82aea0 # Parent 19e26966ed3e7805592801e08d6495855609fac0 Uploaded v0.0.7, depend on Biopython 1.62, tabs to spaces in XML diff -r 19e26966ed3e -r 6842c0c7bc70 tools/filters/repository_dependencies.xml --- a/tools/filters/repository_dependencies.xml Mon Jul 29 09:13:13 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,6 +0,0 @@ - - - - - diff -r 19e26966ed3e -r 6842c0c7bc70 tools/filters/seq_select_by_id.py --- a/tools/filters/seq_select_by_id.py Mon Jul 29 09:13:13 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,130 +0,0 @@ -#!/usr/bin/env python -"""Select FASTA, QUAL, FASTQ or SSF sequences by IDs from a tabular file. - -Takes five command line options, tabular filename, ID column number (using -one based counting), input filename, input type (e.g. FASTA or SFF) and the -output filename (same format as input sequence file). - -When selecting from an SFF file, any Roche XML manifest in the input file is -preserved in both output files. - -This tool is a short Python script which requires Biopython 1.54 or later -for SFF file support. If you use this tool in scientific work leading to a -publication, please cite the Biopython application note: - -Cock et al 2009. Biopython: freely available Python tools for computational -molecular biology and bioinformatics. Bioinformatics 25(11) 1422-3. -http://dx.doi.org/10.1093/bioinformatics/btp163 pmid:19304878. - -This script is copyright 2011-2013 by Peter Cock, The James Hutton Institute UK. -All rights reserved. See accompanying text file for licence details (MIT -license). - -This is version 0.0.6 of the script. -""" -import sys - -def stop_err(msg, err=1): - sys.stderr.write(msg.rstrip() + "\n") - sys.exit(err) - -if "-v" in sys.argv or "--version" in sys.argv: - print "v0.0.6" - sys.exit(0) - -#Parse Command Line -try: - tabular_file, col_arg, in_file, seq_format, out_file = sys.argv[1:] -except ValueError: - stop_err("Expected five arguments, got %i:\n%s" % (len(sys.argv)-1, " ".join(sys.argv))) -try: - if col_arg.startswith("c"): - column = int(col_arg[1:])-1 - else: - column = int(col_arg)-1 -except ValueError: - stop_err("Expected column number, got %s" % col_arg) - -if seq_format == "fastqcssanger": - stop_err("Colorspace FASTQ not supported.") -elif seq_format.lower() in ["sff", "fastq", "qual", "fasta"]: - seq_format = seq_format.lower() -elif seq_format.lower().startswith("fastq"): - #We don't care how the qualities are encoded - seq_format = "fastq" -elif seq_format.lower().startswith("qual"): - #We don't care what the scores are - seq_format = "qual" -else: - stop_err("Unrecognised file format %r" % seq_format) - - -try: - from Bio import SeqIO -except ImportError: - stop_err("Biopython 1.54 or later is required") - - -def parse_ids(tabular_file, col): - """Read tabular file and record all specified identifiers.""" - handle = open(tabular_file, "rU") - for line in handle: - if line.strip() and not line.startswith("#"): - yield line.rstrip("\n").split("\t")[col].strip() - handle.close() - -#Index the sequence file. -#If very big, could use SeqIO.index_db() to avoid memory bottleneck... -records = SeqIO.index(in_file, seq_format) -print "Indexed %i sequences" % len(records) - -if seq_format.lower()=="sff": - #Special case to try to preserve the XML manifest - try: - from Bio.SeqIO.SffIO import SffIterator, SffWriter - except ImportError: - stop_err("Requires Biopython 1.54 or later") - - try: - from Bio.SeqIO.SffIO import ReadRocheXmlManifest - except ImportError: - #Prior to Biopython 1.56 this was a private function - from Bio.SeqIO.SffIO import _sff_read_roche_index_xml as ReadRocheXmlManifest - - in_handle = open(in_file, "rb") #must be binary mode! - try: - manifest = ReadRocheXmlManifest(in_handle) - except ValueError: - manifest = None - in_handle.close() - - out_handle = open(out_file, "wb") - writer = SffWriter(out_handle, xml=manifest) - count = 0 - #This does have the overhead of parsing into SeqRecord objects, - #but doing the header and index at the low level is too fidly. - iterator = (records[name] for name in parse_ids(tabular_file, column)) - try: - count = writer.write_file(iterator) - except KeyError, err: - out_handle.close() - if name not in records: - stop_err("Identifier %r not found in sequence file" % name) - else: - raise err - out_handle.close() -else: - #Avoid overhead of parsing into SeqRecord objects, - #just re-use the original formatting from the input file. - out_handle = open(out_file, "w") - count = 0 - for name in parse_ids(tabular_file, column): - try: - out_handle.write(records.get_raw(name)) - except KeyError: - out_handle.close() - stop_err("Identifier %r not found in sequence file" % name) - count += 1 - out_handle.close() - -print "Selected %i sequences by ID" % count diff -r 19e26966ed3e -r 6842c0c7bc70 tools/filters/seq_select_by_id.rst --- a/tools/filters/seq_select_by_id.rst Mon Jul 29 09:13:13 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,118 +0,0 @@ -Galaxy tool to select FASTA, QUAL, FASTQ or SFF sequences by ID -=============================================================== - -This tool is copyright 2011-2013 by Peter Cock, The James Hutton Institute -(formerly SCRI, Scottish Crop Research Institute), UK. All rights reserved. -See the licence text below. - -This tool is a short Python script (using Biopython library functions) to extract -sequences from a FASTA, QUAL, FASTQ, or SFF file based on the list of IDs given -by a column of a tabular file. The output order follows that of the tabular file, -and if there are duplicates in the tabular file, there will be duplicates in the -output sequence file. - -This tool is available from the Galaxy Tool Shed at: - -* http://toolshed.g2.bx.psu.edu/view/peterjc/seq_select_by_id - -See also the sister tools to filter sequence files according to IDs from column(s) -of a tabular file (where the output order follows the sequence file, and any -duplicate IDs are ignored) and rename sequences: - -* http://toolshed.g2.bx.psu.edu/view/peterjc/seq_filter_by_id -* http://toolshed.g2.bx.psu.edu/view/peterjc/seq_rename - - -Automated Installation -====================== - -This should be straightforward using the Galaxy Tool Shed, which should be -able to automatically install the dependency on Biopython, and then install -this tool and run its unit tests. - - -Manual Installation -=================== - -There are just two files to install to use this tool from within Galaxy: - -* seq_select_by_id.py (the Python script) -* seq_select_by_id.xml (the Galaxy tool definition) - -The suggested location is in the Galaxy folder tools/filters next to the tool -for calling sff_extract.py for converting SFF to FASTQ or FASTA + QUAL. - -You will also need to modify the tools_conf.xml file to tell Galaxy to offer the -tool. One suggested location is in the filters section. Simply add the line:: - - - -If you wish to run the unit tests, also add this to tools_conf.xml.sample -and move/copy the test-data files under Galaxy's test-data folder. Then:: - - $ ./run_functional_tests.sh -id seq_select_by_id - -You will also need to install Biopython 1.54 or later. That's it. - - -History -======= - -======= ====================================================================== -Version Changes -------- ---------------------------------------------------------------------- -v0.0.1 - Initial version. -v0.0.3 - Ignore blank lines in input. -v0.0.4 - Record script version when run from Galaxy. - - Basic unit test included. -v0.0.5 - Check for errors using Python script's return code. -v0.0.6 - Link to Tool Shed added to help text and this documentation. - - Automatic installation of Biopython dependency. - - Use reStructuredText for this README file. - - Adopt standard MIT License. -======= ====================================================================== - - -Developers -========== - -This script and related tools are being developed on the following hg branch: -http://bitbucket.org/peterjc/galaxy-central/src/tools - -For making the "Galaxy Tool Shed" http://toolshed.g2.bx.psu.edu/ tarball use -the following command from the Galaxy root folder:: - - $ tar -czf seq_select_by_id.tar.gz tools/filters/seq_select_by_id.* tools/filters/repository_dependencies.xml test-data/k12_ten_proteins.fasta test-data/k12_hypothetical.fasta test-data/k12_hypothetical.tabular - -Check this worked:: - - $ tar -tzf seq_select_by_id.tar.gz - tools/filters/seq_select_by_id.py - tools/filters/seq_select_by_id.rst - tools/filter/seq_select_by_id.xml - tools/filters/repository_dependencies.xml - test-data/k12_ten_proteins.fasta - test-data/k12_hypothetical.fasta - test-data/k12_hypothetical.tabular - - -Licence (MIT) -============= - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff -r 19e26966ed3e -r 6842c0c7bc70 tools/filters/seq_select_by_id.xml --- a/tools/filters/seq_select_by_id.xml Mon Jul 29 09:13:13 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,65 +0,0 @@ - - from a tabular file - seq_select_by_id.py --version - -seq_select_by_id.py $input_tabular $column $input_file $input_file.ext $output_file - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Bio - - - -**What it does** - -Takes a FASTA, QUAL, FASTQ or Standard Flowgram Format (SFF) file and produces a -new sequence file (of the same format) containing only the records with identifiers -in the tabular file (in the order from the tabular file). - -WARNING: If you have any duplicates in the tabular file identifiers, you will get -duplicate sequences in the output. - -**Citation** - -This tool uses Biopython to read, write and index sequence files. If you use -this tool in scientific work leading to a publication, please cite the -Biopython application note (and Galaxy too of course): - -Cock et al 2009. Biopython: freely available Python tools for computational -molecular biology and bioinformatics. Bioinformatics 25(11) 1422-3. -http://dx.doi.org/10.1093/bioinformatics/btp163 pmid:19304878. - -This tool is available to install into other Galaxy Instances via the Galaxy -Tool Shed at http://toolshed.g2.bx.psu.edu/view/peterjc/seq_select_by_id - - diff -r 19e26966ed3e -r 6842c0c7bc70 tools/seq_select_by_id/README.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tools/seq_select_by_id/README.rst Mon Oct 28 05:21:45 2013 -0400 @@ -0,0 +1,124 @@ +Galaxy tool to select FASTA, QUAL, FASTQ or SFF sequences by ID +=============================================================== + +This tool is copyright 2011-2013 by Peter Cock, The James Hutton Institute +(formerly SCRI, Scottish Crop Research Institute), UK. All rights reserved. +See the licence text below. + +This tool is a short Python script (using Biopython library functions) to extract +sequences from a FASTA, QUAL, FASTQ, or SFF file based on the list of IDs given +by a column of a tabular file. The output order follows that of the tabular file, +and if there are duplicates in the tabular file, there will be duplicates in the +output sequence file. + +This tool is available from the Galaxy Tool Shed at: + +* http://toolshed.g2.bx.psu.edu/view/peterjc/seq_select_by_id + +See also the sister tools to filter sequence files according to IDs from column(s) +of a tabular file (where the output order follows the sequence file, and any +duplicate IDs are ignored) and rename sequences: + +* http://toolshed.g2.bx.psu.edu/view/peterjc/seq_filter_by_id +* http://toolshed.g2.bx.psu.edu/view/peterjc/seq_rename + + +Automated Installation +====================== + +This should be straightforward using the Galaxy Tool Shed, which should be +able to automatically install the dependency on Biopython, and then install +this tool and run its unit tests. + + +Manual Installation +=================== + +There are just two files to install to use this tool from within Galaxy: + +* seq_select_by_id.py (the Python script) +* seq_select_by_id.xml (the Galaxy tool definition) + +The suggested location is a dedicated tools/seq_select_by_id folder. + +You will also need to modify the tools_conf.xml file to tell Galaxy to offer the +tool. One suggested location is in the filters section. Simply add the line:: + + + +If you wish to run the unit tests, also add this to tools_conf.xml.sample +and move/copy the test-data files under Galaxy's test-data folder. Then:: + + $ ./run_functional_tests.sh -id seq_select_by_id + +You will also need to install Biopython 1.54 or later. That's it. + + +History +======= + +======= ====================================================================== +Version Changes +------- ---------------------------------------------------------------------- +v0.0.1 - Initial version. +v0.0.3 - Ignore blank lines in input. +v0.0.4 - Record script version when run from Galaxy. + - Basic unit test included. +v0.0.5 - Check for errors using Python script's return code. +v0.0.6 - Link to Tool Shed added to help text and this documentation. + - Automatic installation of Biopython dependency. + - Use reStructuredText for this README file. + - Adopt standard MIT License. +v0.0.7 - Updated citation information (Cock et al. 2013). + - Fixed Biopython dependency setup. + - Development moved to GitHub, https://github.com/peterjc/pico_galaxy + - Renamed folder and adopted README.rst naming. +======= ====================================================================== + + +Developers +========== + +This script and related tools were initially developed on the following hg branch: +http://bitbucket.org/peterjc/galaxy-central/src/tools + +Development has now moved to a dedicated GitHub repository: +https://github.com/peterjc/pico_galaxy/tree/master/tools + +For making the "Galaxy Tool Shed" http://toolshed.g2.bx.psu.edu/ tarball use +the following command from the Galaxy root folder:: + + $ tar -czf seq_select_by_id.tar.gz tools/seq_select_by_id/README.rst tools/seq_select_by_id/seq_select_by_id.* tools/seq_select_by_id/repository_dependencies.xml test-data/k12_ten_proteins.fasta test-data/k12_hypothetical.fasta test-data/k12_hypothetical.tabular + +Check this worked:: + + $ tar -tzf seq_select_by_id.tar.gz + tools/seq_select_by_id/README.rst + tools/seq_select_by_id/seq_select_by_id.py + tools/seq_select_by_id/seq_select_by_id.xml + tools/seq_select_by_id/repository_dependencies.xml + test-data/k12_ten_proteins.fasta + test-data/k12_hypothetical.fasta + test-data/k12_hypothetical.tabular + + +Licence (MIT) +============= + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff -r 19e26966ed3e -r 6842c0c7bc70 tools/seq_select_by_id/repository_dependencies.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tools/seq_select_by_id/repository_dependencies.xml Mon Oct 28 05:21:45 2013 -0400 @@ -0,0 +1,6 @@ + + + + + diff -r 19e26966ed3e -r 6842c0c7bc70 tools/seq_select_by_id/seq_select_by_id.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tools/seq_select_by_id/seq_select_by_id.py Mon Oct 28 05:21:45 2013 -0400 @@ -0,0 +1,130 @@ +#!/usr/bin/env python +"""Select FASTA, QUAL, FASTQ or SSF sequences by IDs from a tabular file. + +Takes five command line options, tabular filename, ID column number (using +one based counting), input filename, input type (e.g. FASTA or SFF) and the +output filename (same format as input sequence file). + +When selecting from an SFF file, any Roche XML manifest in the input file is +preserved in both output files. + +This tool is a short Python script which requires Biopython 1.54 or later +for SFF file support. If you use this tool in scientific work leading to a +publication, please cite the Biopython application note: + +Cock et al 2009. Biopython: freely available Python tools for computational +molecular biology and bioinformatics. Bioinformatics 25(11) 1422-3. +http://dx.doi.org/10.1093/bioinformatics/btp163 pmid:19304878. + +This script is copyright 2011-2013 by Peter Cock, The James Hutton Institute UK. +All rights reserved. See accompanying text file for licence details (MIT +license). + +This is version 0.0.6 of the script. +""" +import sys + +def stop_err(msg, err=1): + sys.stderr.write(msg.rstrip() + "\n") + sys.exit(err) + +if "-v" in sys.argv or "--version" in sys.argv: + print "v0.0.6" + sys.exit(0) + +#Parse Command Line +try: + tabular_file, col_arg, in_file, seq_format, out_file = sys.argv[1:] +except ValueError: + stop_err("Expected five arguments, got %i:\n%s" % (len(sys.argv)-1, " ".join(sys.argv))) +try: + if col_arg.startswith("c"): + column = int(col_arg[1:])-1 + else: + column = int(col_arg)-1 +except ValueError: + stop_err("Expected column number, got %s" % col_arg) + +if seq_format == "fastqcssanger": + stop_err("Colorspace FASTQ not supported.") +elif seq_format.lower() in ["sff", "fastq", "qual", "fasta"]: + seq_format = seq_format.lower() +elif seq_format.lower().startswith("fastq"): + #We don't care how the qualities are encoded + seq_format = "fastq" +elif seq_format.lower().startswith("qual"): + #We don't care what the scores are + seq_format = "qual" +else: + stop_err("Unrecognised file format %r" % seq_format) + + +try: + from Bio import SeqIO +except ImportError: + stop_err("Biopython 1.54 or later is required") + + +def parse_ids(tabular_file, col): + """Read tabular file and record all specified identifiers.""" + handle = open(tabular_file, "rU") + for line in handle: + if line.strip() and not line.startswith("#"): + yield line.rstrip("\n").split("\t")[col].strip() + handle.close() + +#Index the sequence file. +#If very big, could use SeqIO.index_db() to avoid memory bottleneck... +records = SeqIO.index(in_file, seq_format) +print "Indexed %i sequences" % len(records) + +if seq_format.lower()=="sff": + #Special case to try to preserve the XML manifest + try: + from Bio.SeqIO.SffIO import SffIterator, SffWriter + except ImportError: + stop_err("Requires Biopython 1.54 or later") + + try: + from Bio.SeqIO.SffIO import ReadRocheXmlManifest + except ImportError: + #Prior to Biopython 1.56 this was a private function + from Bio.SeqIO.SffIO import _sff_read_roche_index_xml as ReadRocheXmlManifest + + in_handle = open(in_file, "rb") #must be binary mode! + try: + manifest = ReadRocheXmlManifest(in_handle) + except ValueError: + manifest = None + in_handle.close() + + out_handle = open(out_file, "wb") + writer = SffWriter(out_handle, xml=manifest) + count = 0 + #This does have the overhead of parsing into SeqRecord objects, + #but doing the header and index at the low level is too fidly. + iterator = (records[name] for name in parse_ids(tabular_file, column)) + try: + count = writer.write_file(iterator) + except KeyError, err: + out_handle.close() + if name not in records: + stop_err("Identifier %r not found in sequence file" % name) + else: + raise err + out_handle.close() +else: + #Avoid overhead of parsing into SeqRecord objects, + #just re-use the original formatting from the input file. + out_handle = open(out_file, "w") + count = 0 + for name in parse_ids(tabular_file, column): + try: + out_handle.write(records.get_raw(name)) + except KeyError: + out_handle.close() + stop_err("Identifier %r not found in sequence file" % name) + count += 1 + out_handle.close() + +print "Selected %i sequences by ID" % count diff -r 19e26966ed3e -r 6842c0c7bc70 tools/seq_select_by_id/seq_select_by_id.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tools/seq_select_by_id/seq_select_by_id.xml Mon Oct 28 05:21:45 2013 -0400 @@ -0,0 +1,72 @@ + + from a tabular file + + biopython + Bio + + seq_select_by_id.py --version + +seq_select_by_id.py $input_tabular $column $input_file $input_file.ext $output_file + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +**What it does** + +Takes a FASTA, QUAL, FASTQ or Standard Flowgram Format (SFF) file and produces a +new sequence file (of the same format) containing only the records with identifiers +in the tabular file (in the order from the tabular file). + +WARNING: If you have any duplicates in the tabular file identifiers, you will get +duplicate sequences in the output. + +**References** + +If you use this Galaxy tool in work leading to a scientific publication please +cite the following papers: + +Peter J.A. Cock, Björn A. Grüning, Konrad Paszkiewicz and Leighton Pritchard (2013). +Galaxy tools and workflows for sequence analysis with applications +in molecular plant pathology. PeerJ 1:e167 +http://dx.doi.org/10.7717/peerj.167 + +This tool uses Biopython to read, write and index sequence files, so you may +also wish to cite the Biopython application note (and Galaxy too of course): + +Cock et al (2009). Biopython: freely available Python tools for computational +molecular biology and bioinformatics. Bioinformatics 25(11) 1422-3. +http://dx.doi.org/10.1093/bioinformatics/btp163 pmid:19304878. + +This tool is available to install into other Galaxy Instances via the Galaxy +Tool Shed at http://toolshed.g2.bx.psu.edu/view/peterjc/seq_select_by_id + +