| Next changeset 1:5a0d0bee4f8d (2019-12-03) |
|
Commit message:
Imported from capsule None |
|
added:
.shed.yml README data_manager/data_manager_diamond_database_builder.py data_manager/data_manager_diamond_database_builder.xml data_manager_conf.xml tool-data/diamond_database.loc.sample tool_data_table_conf.xml.sample tool_dependencies.xml |
| b |
| diff -r 000000000000 -r ce62d0912b10 .shed.yml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/.shed.yml Sun Feb 08 10:05:20 2015 -0500 |
| b |
| @@ -0,0 +1,2 @@ +owner: bgruening +name: data_manager_diamond_database_builder |
| b |
| diff -r 000000000000 -r ce62d0912b10 README --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/README Sun Feb 08 10:05:20 2015 -0500 |
| b |
| @@ -0,0 +1,2 @@ +Diamond database builder +======================== |
| b |
| diff -r 000000000000 -r ce62d0912b10 data_manager/data_manager_diamond_database_builder.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager/data_manager_diamond_database_builder.py Sun Feb 08 10:05:20 2015 -0500 |
| [ |
| b'@@ -0,0 +1,238 @@\n+#!/usr/bin/env python\n+import sys\n+import os\n+import tempfile\n+import shutil\n+import optparse\n+import urllib2\n+import subprocess\n+from ftplib import FTP\n+import tarfile\n+import zipfile\n+import gzip\n+import bz2\n+\n+from galaxy.util.json import from_json_string, to_json_string\n+\n+CHUNK_SIZE = 2**20 #1mb\n+\n+def cleanup_before_exit( tmp_dir ):\n+ if tmp_dir and os.path.exists( tmp_dir ):\n+ shutil.rmtree( tmp_dir )\n+\n+\n+def stop_err(msg):\n+ sys.stderr.write(msg)\n+ sys.exit(1)\n+\n+\n+def _get_files_in_ftp_path( ftp, path ):\n+ path_contents = []\n+ ftp.retrlines( \'MLSD %s\' % ( path ), path_contents.append )\n+ return [ line.split( \';\' )[ -1 ].lstrip() for line in path_contents ]\n+\n+\n+def _get_stream_readers_for_tar( file_obj, tmp_dir ):\n+ fasta_tar = tarfile.open( fileobj=file_obj, mode=\'r:*\' )\n+ return [ fasta_tar.extractfile( member ) for member in fasta_tar.getmembers() ]\n+\n+\n+def _get_stream_readers_for_zip( file_obj, tmp_dir ):\n+ fasta_zip = zipfile.ZipFile( file_obj, \'r\' )\n+ rval = []\n+ for member in fasta_zip.namelist():\n+ fasta_zip.extract( member, tmp_dir )\n+ rval.append( open( os.path.join( tmp_dir, member ), \'rb\' ) )\n+ return rval\n+\n+\n+def _get_stream_readers_for_gzip( file_obj, tmp_dir ):\n+ return [ gzip.GzipFile( fileobj=file_obj, mode=\'rb\' ) ]\n+\n+\n+def _get_stream_readers_for_bz2( file_obj, tmp_dir ):\n+ return [ bz2.BZ2File( file_obj.name, \'rb\' ) ]\n+\n+\n+def download_from_ncbi( data_manager_dict, params, target_directory, database_id, database_name ):\n+ NCBI_FTP_SERVER = \'ftp.ncbi.nlm.nih.gov\'\n+ NCBI_DOWNLOAD_PATH = \'/blast/db/FASTA/\'\n+ COMPRESSED_EXTENSIONS = [ ( \'.tar.gz\', _get_stream_readers_for_tar ), ( \'.tar.bz2\', _get_stream_readers_for_tar ), ( \'.zip\', _get_stream_readers_for_zip ), ( \'.gz\', _get_stream_readers_for_gzip ), ( \'.bz2\', _get_stream_readers_for_bz2 ) ]\n+\n+ ncbi_identifier = params[\'param_dict\'][\'reference_source\'][\'requested_identifier\']\n+ ftp = FTP( NCBI_FTP_SERVER )\n+ ftp.login()\n+\n+ path_contents = _get_files_in_ftp_path( ftp, NCBI_DOWNLOAD_PATH )\n+\n+ ncbi_file_name = None\n+ get_stream_reader = None\n+ ext = None\n+ for ext, get_stream_reader in COMPRESSED_EXTENSIONS:\n+ if "%s%s" % ( ncbi_identifier, ext ) in path_contents:\n+ ncbi_file_name = "%s%s%s" % ( NCBI_DOWNLOAD_PATH, ncbi_identifier, ext )\n+ break\n+\n+ if not ncbi_file_name:\n+ raise Exception( \'Unable to determine filename for NCBI database for %s: %s\' % ( ncbi_identifier, path_contents ) )\n+\n+ tmp_dir = tempfile.mkdtemp( prefix=\'tmp-data-manager-ncbi-\' )\n+ ncbi_fasta_filename = os.path.join( tmp_dir, "%s%s" % ( ncbi_identifier, ext ) )\n+ \n+ fasta_base_filename = "%s.fa" % database_id\n+ fasta_filename = os.path.join( target_directory, fasta_base_filename )\n+ fasta_writer = open( fasta_filename, \'wb+\' )\n+ \n+ tmp_extract_dir = os.path.join ( tmp_dir, \'extracted_fasta\' )\n+ os.mkdir( tmp_extract_dir )\n+ \n+ tmp_fasta = open( ncbi_fasta_filename, \'wb+\' )\n+ \n+ ftp.retrbinary( \'RETR %s\' % ncbi_file_name, tmp_fasta.write )\n+ \n+ tmp_fasta.flush()\n+ tmp_fasta.seek( 0 )\n+ \n+ fasta_readers = get_stream_reader( tmp_fasta, tmp_extract_dir )\n+ \n+ data_table_entry = _stream_fasta_to_file( fasta_readers, target_directory, database_id, database_name, params )\n+ _add_data_table_entry( data_manager_dict, data_table_entry )\n+ \n+ for fasta_reader in fasta_readers:\n+ fasta_reader.close()\n+ tmp_fasta.close()\n+ cleanup_before_exit( tmp_dir )\n+\n+\n+def download_from_url( data_manager_dict, params, target_directory, database_id, database_name ):\n+ #TODO: we should automatically do decompression here\n+ urls = filter( bool, map( lambda x: x.strip(), params[\'param_dict\'][\'reference_source\'][\'user_url\'].split( \'\\n\' ) ) )\n+ fasta_reader = [ urllib2.urlopen( url ) for url in urls ]\n+ \n+ data_table_entry = _stream_fasta_to_file( fasta_rea'..b'able_entry )\n+\n+\n+def _add_data_table_entry( data_manager_dict, data_table_entry ):\n+ data_manager_dict[\'data_tables\'] = data_manager_dict.get( \'data_tables\', {} )\n+ data_manager_dict[\'data_tables\'][\'diamond_database\'] = data_manager_dict[\'data_tables\'].get( \'diamond_database\', [] )\n+ data_manager_dict[\'data_tables\'][\'diamond_database\'].append( data_table_entry )\n+ return data_manager_dict\n+\n+\n+def _stream_fasta_to_file( fasta_stream, target_directory, database_id, database_name, params, close_stream=True ):\n+ fasta_base_filename = "%s.fa" % database_id\n+ fasta_filename = os.path.join( target_directory, fasta_base_filename )\n+\n+ temp_fasta = tempfile.NamedTemporaryFile( delete=False, suffix=".fasta" )\n+ temp_fasta.close()\n+ fasta_writer = open( temp_fasta.name, \'wb+\' )\n+\n+ if isinstance( fasta_stream, list ) and len( fasta_stream ) == 1:\n+ fasta_stream = fasta_stream[0]\n+\n+ if isinstance( fasta_stream, list ):\n+ last_char = None\n+ for fh in fasta_stream:\n+ if last_char not in [ None, \'\\n\', \'\\r\' ]:\n+ fasta_writer.write( \'\\n\' )\n+ while True:\n+ data = fh.read( CHUNK_SIZE )\n+ if data:\n+ fasta_writer.write( data )\n+ last_char = data[-1]\n+ else:\n+ break\n+ if close_stream:\n+ fh.close()\n+ else:\n+ while True:\n+ data = fasta_stream.read( CHUNK_SIZE )\n+ if data:\n+ fasta_writer.write( data )\n+ else:\n+ break\n+ if close_stream:\n+ fasta_stream.close()\n+\n+ fasta_writer.close()\n+\n+ args = [ \'diamond\', \'makedb\', \'--in\', temp_fasta.name, \'--db\', fasta_filename]\n+\n+ tmp_stderr = tempfile.NamedTemporaryFile( prefix = "tmp-data-manager-diamond-database-builder-stderr" )\n+ proc = subprocess.Popen( args=args, shell=False, cwd=target_directory, stderr=tmp_stderr.fileno() )\n+ return_code = proc.wait()\n+ if return_code:\n+ tmp_stderr.flush()\n+ tmp_stderr.seek(0)\n+ print >> sys.stderr, "Error building diamond database:"\n+ while True:\n+ chunk = tmp_stderr.read( CHUNK_SIZE )\n+ if not chunk:\n+ break\n+ sys.stderr.write( chunk )\n+ sys.exit( return_code )\n+ tmp_stderr.close()\n+ os.remove( temp_fasta.name )\n+ return dict( value=database_id, name=database_name, db_path="%s.dmnd" % fasta_base_filename )\n+\n+\n+def _create_symlink( input_filename, target_directory, database_id, database_name ):\n+ fasta_base_filename = "%s.fa" % database_id\n+ fasta_filename = os.path.join( target_directory, fasta_base_filename )\n+ os.symlink( input_filename, fasta_filename )\n+ return dict( value=database_id, name=database_name, db_path=fasta_base_filename )\n+\n+\n+REFERENCE_SOURCE_TO_DOWNLOAD = dict( ncbi=download_from_ncbi, url=download_from_url, history=download_from_history, directory=copy_from_directory )\n+\n+def main():\n+ #Parse Command Line\n+ parser = optparse.OptionParser()\n+ parser.add_option( \'-d\', \'--dbkey_description\', dest=\'dbkey_description\', action=\'store\', type="string", default=None, help=\'dbkey_description\' )\n+ (options, args) = parser.parse_args()\n+ \n+ filename = args[0]\n+ \n+ params = from_json_string( open( filename ).read() )\n+ target_directory = params[ \'output_data\' ][0][\'extra_files_path\']\n+ os.mkdir( target_directory )\n+ data_manager_dict = {}\n+\n+ database_id = params[\'param_dict\'][\'database_id\']\n+ database_name = params[\'param_dict\'][\'database_name\']\n+\n+ #Fetch the FASTA\n+ REFERENCE_SOURCE_TO_DOWNLOAD[ params[\'param_dict\'][\'reference_source\'][\'reference_source_selector\'] ]( data_manager_dict, params, target_directory, database_id, database_name )\n+ \n+ #save info to json file\n+ open( filename, \'wb\' ).write( to_json_string( data_manager_dict ) )\n+\n+if __name__ == "__main__":\n+ main()\n' |
| b |
| diff -r 000000000000 -r ce62d0912b10 data_manager/data_manager_diamond_database_builder.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager/data_manager_diamond_database_builder.xml Sun Feb 08 10:05:20 2015 -0500 |
| b |
| @@ -0,0 +1,46 @@ +<tool id="diamond_database_builder" name="Diamond" tool_type="manage_data" version="0.0.1"> + <description> Database builder</description> + <requirements> + <requirement type="package" version="0.6.13">diamond</requirement> + </requirements> + <command interpreter="python"> + data_manager_diamond_database_builder.py "${out_file}" + </command> + <inputs> + <param type="text" name="database_name" value="" label="Database name or description" + help="This will be shown to the user in the database selectbox."/> + <param type="text" name="database_id" value="" label="Database ID" help="This needs to be unique. No whitespaces allowed. Use underscores." /> + <conditional name="reference_source"> + <param name="reference_source_selector" type="select" label="Choose the source for the reference genome"> + <option value="ncbi">NCBI</option> + <option value="url">URL</option> + <option value="history" selected="True" help="Recommended to archive the original FASTA file as well.">History</option> + <option value="directory">Directory on Server</option> + </param> + <when value="ncbi"> + <param type="text" name="requested_identifier" value="" label="NCBI identifier" help="e.g. nr" optional="False" /> + </when> + <when value="url"> + <param type="text" area="True" name="user_url" value="http://" label="URLs" optional="False" /> + </when> + <when value="history"> + <param name="input_fasta" type="data" format="fasta" label="FASTA File" multiple="False" optional="False" /> + </when> + <when value="directory"> + <param type="text" name="fasta_filename" value="" label="Full path to FASTA File on disk" optional="False" /> + <param type="boolean" name="create_symlink" truevalue="create_symlink" falsevalue="copy_file" label="Create symlink to orignal data instead of copying" checked="False" /> + </when> + </conditional> + </inputs> + <outputs> + <data name="out_file" format="data_manager_json"/> + </outputs> + <help> + +.. class:: infomark + +NCBI databases can be downded from ftp://ftp.ncbi.nlm.nih.gov/blast/db/FASTA/ +For example the NR database can be downloaded from ftp://ftp.ncbi.nlm.nih.gov/blast/db/FASTA/nr.gz + + </help> +</tool> |
| b |
| diff -r 000000000000 -r ce62d0912b10 data_manager_conf.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager_conf.xml Sun Feb 08 10:05:20 2015 -0500 |
| b |
| @@ -0,0 +1,19 @@ +<?xml version="1.0"?> +<data_managers> + <data_manager tool_file="data_manager/data_manager_diamond_database_builder.xml" id="diamond_database_builder" version="0.0.1"> + <data_table name="diamond_database"> + <output> + <column name="value" /> + <column name="name" /> + <column name="db_path" output_ref="out_file" > + <move type="directory" relativize_symlinks="True"> + <!-- <source>${db_path}</source>--> <!-- out_file.extra_files_path is used as base by default --> <!-- if no source, eg for type=directory, then refers to base --> + <target base="${GALAXY_DATA_MANAGER_DATA_PATH}">diamond_database/${value}</target> + </move> + <value_translation>${GALAXY_DATA_MANAGER_DATA_PATH}/diamond_database/${value}/${db_path}</value_translation> + <value_translation type="function">abspath</value_translation> + </column> + </output> + </data_table> + </data_manager> +</data_managers> |
| b |
| diff -r 000000000000 -r ce62d0912b10 tool-data/diamond_database.loc.sample --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool-data/diamond_database.loc.sample Sun Feb 08 10:05:20 2015 -0500 |
| b |
| @@ -0,0 +1,18 @@ +#This file lists the locations and dbkeys of all the fasta files +#under the "genome" directory (a directory that contains a directory +#for each build). The script extract_fasta.py will generate the file +#all_fasta.loc. This file has the format (white space characters are +#TAB characters): +# +#<unique_build_id> <dbkey> <display_name> <file_path> +# +#So, all_fasta.loc could look something like this: +# +#apiMel3 apiMel3 Honeybee (Apis mellifera): apiMel3 /path/to/genome/apiMel3/apiMel3.fa +#hg19canon hg19 Human (Homo sapiens): hg19 Canonical /path/to/genome/hg19/hg19canon.fa +#hg19full hg19 Human (Homo sapiens): hg19 Full /path/to/genome/hg19/hg19full.fa +# +#Your all_fasta.loc file should contain an entry for each individual +#fasta file. So there will be multiple fasta files for each build, +#such as with hg19 above. +# |
| b |
| diff -r 000000000000 -r ce62d0912b10 tool_data_table_conf.xml.sample --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool_data_table_conf.xml.sample Sun Feb 08 10:05:20 2015 -0500 |
| b |
| @@ -0,0 +1,7 @@ +<tables> + <!-- Locations of all fasta files required to build Diamond databases --> + <table name="diamond_database" comment_char="#"> + <columns>value, name, db_path</columns> + <file path="tool-data/diamond_database.loc" /> + </table> +</tables> |
| b |
| diff -r 000000000000 -r ce62d0912b10 tool_dependencies.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool_dependencies.xml Sun Feb 08 10:05:20 2015 -0500 |
| b |
| @@ -0,0 +1,6 @@ +<?xml version="1.0"?> +<tool_dependency> + <package name="diamond" version="0.6.13"> + <repository changeset_revision="47c4dfd5aed5" name="package_diamond_0_6_13" owner="iuc" toolshed="https://toolshed.g2.bx.psu.edu" /> + </package> +</tool_dependency> |